[TASK] add ciUpload cmd to uload files based on .cihsignore file

Author: dmh

lib/cmd/ciUpload.exec.js

@@ -0,0 +1,2 @@
+import { ciUpload } from './ciUpload.js'
+ciUpload()

lib/cmd/ciUpload.js

@@ -0,0 +1,45 @@
+import '../utils/check.js'
+import { themeName } from '../hubspot/theme.js'
+import { loadAuthConfig } from '../hubspot/auth/auth.js'
+import { ciUploadTheme } from '../hubspot/upload.js'
+import { cleanVendorDestFolder } from '../compile/clean.js'
+import { compileScss } from '../compile/sass.js'
+import { compileCss } from '../compile/css.js'
+import { compileJs } from '../compile/js.js'
+import { compileFieldsJs } from '../hubspot/fields.js'
+import * as ui from '../utils/ui.js'
+import chalk from 'chalk'
+
+/**
+ * #### CI Upload theme to HUBSPOT cms portall
+ * @async
+ * @memberof Commands
+ * @example
+ * // run with node
+ * echo "import '@resultify/hubspot-cms-lib/ciUpload'" > ciUpload.js
+ * node ciUpload.js
+ *
+ *
+ * // run with npm by adding to package.json scripts:
+ * "scripts": {
+ *  "ciUpload": "cmslib --ciUpload",
+ * }
+ * or
+ * "scripts": {
+ *   "ciUpload": "node -e 'import(`@resultify/hubspot-cms-lib/ciUpload`)'"
+ * }
+ * npm run ciUpload
+ */
+async function ciUpload () {
+  const timeStart = ui.startTaskGroup(`Clean upload ${chalk.yellow(themeName)}`)
+  const hubAuthData = await loadAuthConfig()
+  await cleanVendorDestFolder()
+  await compileScss()
+  await compileCss()
+  await compileJs()
+  await compileFieldsJs()
+  await ciUploadTheme(hubAuthData, themeName)
+  ui.endTaskGroup({ taskName: `Clean upload ${chalk.yellow(themeName)}`, timeStart })
+}
+
+export { ciUpload }

lib/hubspot/upload.js

@@ -1,5 +1,6 @@
 /** @module hubspot/upload */
 /// <reference path="../types/types.js" />
+import fsPromises from 'fs/promises'
 import uploadFolder from '@hubspot/local-dev-lib/cms/uploadFolder'
 import { deleteFile, getDirectoryContentsByPath } from '@hubspot/local-dev-lib/api/fileMapper'
 import { setLogLevel, LOG_LEVEL } from '@hubspot/local-dev-lib/logger'
@@ -9,6 +10,7 @@ import { walk } from '@hubspot/local-dev-lib/fs'
 import * as ui from '../utils/ui.js'
 import { getThemeOptions } from '../utils/options.js'
 import { throwErrorIfMissingScope } from './auth/scopes.js'
+import { isFileDir } from '../utils/fs.js'
 import ora from 'ora'
 import chalk from 'chalk'
 
@@ -47,6 +49,59 @@ const getUploadableFileList = async (src) => {
   return allowedFiles
 }
 
+/**
+ * #### Reads the .cihsignore file and returns the list of files to ignore.
+ * @private
+ * @async
+ * @returns {Promise<string[]>} List of files to ignore.
+ */
+async function readCihsIgnore () {
+  try {
+    if (await isFileDir(`${process.cwd()}/.cihsignore`)) {
+      const ignoreContent = await fsPromises.readFile(`${process.cwd()}/.cihsignore`, 'utf8')
+      return ignoreContent.split('\n').map(line => line.trim()).filter(line => line && !line.startsWith('#'))
+    } else {
+      return []
+    }
+  } catch (error) {
+    console.error('Error reading .cihsignore file:', error)
+    return []
+  }
+}
+
+/**
+ * #### Walks the src folder for files, filters them based on ignore filter and CI .cihsignore file.
+ * @async
+ * @private
+ * @param {string} src - src folder
+ * @returns {Promise<Array<string>>} src file list
+ */
+const getCiUploadableFileList = async (src) => {
+  /**
+   * @type {Array<string>}
+   */
+  let filePaths = []
+  try {
+    filePaths = await walk(src)
+  } catch (error) {
+    console.error(error)
+  }
+  let allowedFiles = filePaths.filter((/** @type {any} */ file) => {
+    if (!isAllowedExtension(file)) {
+      return false
+    }
+    return true
+    // @ts-ignore
+  }).filter(createIgnoreFilter())
+
+  async function filterFiles (/** @type {any} */ files) {
+    const ignoredPaths = await readCihsIgnore()
+    return files.filter((/** @type {any} */ file) => !ignoredPaths.some(ignoredPath => file.includes(ignoredPath)))
+  }
+  allowedFiles = await filterFiles(allowedFiles)
+  return allowedFiles
+}
+
 /**
  * #### upload all HubSpot theme files
  * @async
@@ -111,8 +166,8 @@ async function cleanUploadThemeTemplates (config, themeName) {
     setLogLevel(LOG_LEVEL.NONE)
     const uploadableFileList = await getUploadableFileList(src)
     const filesToDelete = await getDirectoryContentsByPath(portalId, dest)
-    if (filesToDelete !== undefined && filesToDelete.children.length > 0) {
-      for await (const file of filesToDelete.children) {
+    if (filesToDelete !== undefined && filesToDelete.data.children.length > 0) {
+      for await (const file of filesToDelete.data.children) {
         // @ts-ignore
         if (file.includes('.html')) {
           await deleteFile(portalId, `${dest}/${file}`)
@@ -168,8 +223,8 @@ async function cleanUploadTheme (config, themeName) {
     const uploadableFileList = await getUploadableFileList(src)
     spinner.start()
     const filesToDelete = await getDirectoryContentsByPath(portalId, '/')
-    if (filesToDelete !== undefined && filesToDelete.children.length > 0) {
-      for await (const file of filesToDelete.children) {
+    if (filesToDelete !== undefined && filesToDelete.data.children.length > 0) {
+      for await (const file of filesToDelete.data.children) {
         // @ts-ignore
         if (file === dest) {
           await deleteFile(portalId, file)
@@ -204,4 +259,61 @@ async function cleanUploadTheme (config, themeName) {
   }
 }
 
-export { uploadTheme, cleanUploadThemeTemplates, cleanUploadTheme }
+/**
+ * #### CI clean theme upload
+ * @async
+ * @param {HUBSPOT_AUTH_CONFIG} config - hubspot authentication config
+ * @param {string} themeName - theme name
+ * @returns undefined
+ */
+async function ciUploadTheme (config, themeName) {
+  const timeStart = ui.startTask('ciUpload')
+  const dest = themeName
+  const spinner = ora(`Clean ${dest} folder before upload`)
+  try {
+    const cmslibOptions = getThemeOptions()
+    const src = `${process.cwd()}/${cmslibOptions.themeFolder}`
+    const portalId = config.portals[0].portalId
+
+    throwErrorIfMissingScope(config, 'design_manager')
+    setLogLevel(LOG_LEVEL.LOG)
+    const uploadableFileList = await getCiUploadableFileList(src)
+    spinner.start()
+    const filesToDelete = await getDirectoryContentsByPath(portalId, '/')
+    if (filesToDelete !== undefined && filesToDelete.data.children.length > 0) {
+      for await (const file of filesToDelete.data.children) {
+        // @ts-ignore
+        if (file === dest) {
+          await deleteFile(portalId, file)
+        }
+      }
+    }
+    spinner.succeed()
+    await uploadFolder.uploadFolder(
+      portalId,
+      src,
+      dest,
+      { overwrite: false },
+      { saveOutput: true, convertFields: false },
+      uploadableFileList,
+      cmsMode
+    )
+    ui.endTask({ taskName: 'ciUpload', timeStart })
+  } catch (error) {
+    spinner.fail()
+    if (error.cause.response.data.errorType === 'TEMPLATE_VALIDATION_FAILED') {
+      console.error(chalk.red(error.cause.response.data.errorType))
+      console.error(error.message)
+      process.exitCode = 1
+    } else {
+      console.error(error.message)
+      process.exitCode = 1
+    }
+    if (process.env.DEBUG_MODE === 'debug') {
+      console.error(error)
+      process.exitCode = 1
+    }
+  }
+}
+
+export { uploadTheme, cleanUploadThemeTemplates, cleanUploadTheme, ciUploadTheme }

lib/init.js

@@ -4,6 +4,7 @@ import { watch } from './cmd/watch.js'
 import { hubspotFetchAll } from './cmd/fetch.js'
 import { hubspotFetchModules } from './cmd/fetchModules.js'
 import { upload } from './cmd/upload.js'
+import { ciUpload } from './cmd/ciUpload.js'
 import { cleanUpload } from './cmd/cleanUpload.js'
 import { validate } from './cmd/validate.js'
 import { lighthouse } from './cmd/lighthouse.js'
@@ -23,6 +24,8 @@ function init () {
     hubspotFetchModules()
   } else if (args.upload) {
     upload()
+  } else if (args.ciUpload) {
+    ciUpload()
   } else if (args.cleanUpload) {
     cleanUpload()
   } else if (args.validate) {

lib/utils/arg.js

@@ -10,6 +10,7 @@ const argOptions = {
     'fetch',
     'fetchModules',
     'upload',
+    'ciUpload',
     'cleanUpload',
     'validate',
     'lighthouse',