add server to monorepo.

packages/server/src/lib/Seeder/FsMigrations.ts

@@ -0,0 +1,100 @@
+import path from 'path';
+import { sortBy } from 'lodash';
+import fs from 'fs';
+import { promisify } from 'util';
+import { MigrateItem } from './interfaces';
+import { importWebpackSeedModule } from './Utils';
+import { DEFAULT_LOAD_EXTENSIONS } from './constants';
+import { filterMigrations } from './MigrateUtils';
+
+const readdir = promisify(fs.readdir);
+
+class FsMigrations {
+  private sortDirsSeparately: boolean;
+  private migrationsPaths: string[];
+  private loadExtensions: string[];
+
+  /**
+   * Constructor method.
+   * @param migrationDirectories
+   * @param sortDirsSeparately
+   * @param loadExtensions
+   */
+  constructor(
+    migrationDirectories: string[],
+    sortDirsSeparately: boolean,
+    loadExtensions: string[]
+  ) {
+    this.sortDirsSeparately = sortDirsSeparately;
+
+    if (!Array.isArray(migrationDirectories)) {
+      migrationDirectories = [migrationDirectories];
+    }
+    this.migrationsPaths = migrationDirectories;
+    this.loadExtensions = loadExtensions || DEFAULT_LOAD_EXTENSIONS;
+  }
+
+  /**
+   * Gets the migration names
+   * @returns Promise<MigrateItem[]>
+   */
+  public getMigrations(loadExtensions = null): Promise<MigrateItem[]> {
+    // Get a list of files in all specified migration directories
+    const readMigrationsPromises = this.migrationsPaths.map((configDir) => {
+      const absoluteDir = path.resolve(process.cwd(), configDir);
+      return readdir(absoluteDir).then((files) => ({
+        files,
+        configDir,
+        absoluteDir,
+      }));
+    });
+
+    return Promise.all(readMigrationsPromises).then((allMigrations) => {
+      const migrations = allMigrations.reduce((acc, migrationDirectory) => {
+        // When true, files inside the folder should be sorted
+        if (this.sortDirsSeparately) {
+          migrationDirectory.files = migrationDirectory.files.sort();
+        }
+        migrationDirectory.files.forEach((file) =>
+          acc.push({ file, directory: migrationDirectory.configDir })
+        );
+        return acc;
+      }, []);
+
+      // If true we have already sorted the migrations inside the folders
+      // return the migrations fully qualified
+      if (this.sortDirsSeparately) {
+        return filterMigrations(
+          this,
+          migrations,
+          loadExtensions || this.loadExtensions
+        );
+      }
+      return filterMigrations(
+        this,
+        sortBy(migrations, 'file'),
+        loadExtensions || this.loadExtensions
+      );
+    });
+  }
+
+  /**
+   * Retrieve the file name from given migrate item.
+   * @param {MigrateItem} migration
+   * @returns {string}
+   */
+  public getMigrationName(migration: MigrateItem): string {
+    return migration.file;
+  }
+
+  /**
+   * Retrieve the migrate file content from given migrate item.
+   * @param {MigrateItem} migration
+   * @returns {string}
+   */
+  public getMigration(migration: MigrateItem): string {
+    return importWebpackSeedModule(migration.file);
+  }
+}
+
+export { DEFAULT_LOAD_EXTENSIONS, FsMigrations };

packages/server/src/lib/Seeder/MigrateUtils.ts

@@ -0,0 +1,192 @@
+import { differenceWith } from 'lodash';
+import path from 'path';
+import { FsMigrations } from './FsMigrations';
+import {
+  getTable,
+  getTableName,
+  getLockTableName,
+  getLockTableNameWithSchema,
+} from './TableUtils';
+import { ISeederConfig, MigrateItem } from './interfaces';
+
+/**
+ * Get schema-aware schema builder for a given schema nam
+ * @param trxOrKnex
+ * @param {string} schemaName
+ * @returns
+ */
+function getSchemaBuilder(trxOrKnex, schemaName: string | null = null) {
+  return schemaName
+    ? trxOrKnex.schema.withSchema(schemaName)
+    : trxOrKnex.schema;
+}
+
+/**
+ * Creates migration table of the given table name.
+ * @param {string} tableName
+ * @param {string} schemaName
+ * @param trxOrKnex
+ * @returns
+ */
+function createMigrationTable(
+  tableName: string,
+  schemaName: string,
+  trxOrKnex
+) {
+  return getSchemaBuilder(trxOrKnex, schemaName).createTable(
+    getTableName(tableName),
+    (t) => {
+      t.increments();
+      t.string('name');
+      t.integer('batch');
+      t.timestamp('migration_time');
+    }
+  );
+}
+
+/**
+ * Creates a migration lock table of the given table name.
+ * @param {string} tableName
+ * @param {string} schemaName
+ * @param trxOrKnex
+ * @returns
+ */
+function createMigrationLockTable(
+  tableName: string,
+  schemaName: string,
+  trxOrKnex
+) {
+  return getSchemaBuilder(trxOrKnex, schemaName).createTable(tableName, (t) => {
+    t.increments('index').primary();
+    t.integer('is_locked');
+  });
+}
+
+/**
+ *
+ * @param tableName
+ * @param schemaName
+ * @param trxOrKnex
+ * @returns
+ */
+export function ensureMigrationTables(
+  tableName: string,
+  schemaName: string,
+  trxOrKnex
+) {
+  const lockTable = getLockTableName(tableName);
+  const lockTableWithSchema = getLockTableNameWithSchema(tableName, schemaName);
+
+  return getSchemaBuilder(trxOrKnex, schemaName)
+    .hasTable(tableName)
+    .then((exists) => {
+      return !exists && createMigrationTable(tableName, schemaName, trxOrKnex);
+    })
+    .then(() => {
+      return getSchemaBuilder(trxOrKnex, schemaName).hasTable(lockTable);
+    })
+    .then((exists) => {
+      return (
+        !exists && createMigrationLockTable(lockTable, schemaName, trxOrKnex)
+      );
+    })
+    .then(() => {
+      return getTable(trxOrKnex, lockTable, schemaName).select('*');
+    })
+    .then((data) => {
+      return (
+        !data.length &&
+        trxOrKnex.into(lockTableWithSchema).insert({ is_locked: 0 })
+      );
+    });
+}
+
+/**
+ * Lists all available migration versions, as a sorted array.
+ * @param migrationSource
+ * @param loadExtensions
+ * @returns
+ */
+function listAll(
+  migrationSource: FsMigrations,
+  loadExtensions
+): Promise<MigrateItem[]> {
+  return migrationSource.getMigrations(loadExtensions);
+}
+
+/**
+ * Lists all migrations that have been completed for the current db, as an array.
+ * @param {string} tableName
+ * @param {string} schemaName
+ * @param {} trxOrKnex
+ * @returns Promise<string[]>
+ */
+export async function listCompleted(
+  tableName: string,
+  schemaName: string,
+  trxOrKnex
+): Promise<string[]> {
+  const completedMigrations = await trxOrKnex
+    .from(getTableName(tableName, schemaName))
+    .orderBy('id')
+    .select('name');
+
+  return completedMigrations.map((migration) => {
+    return migration.name;
+  });
+}
+
+/**
+ * Gets the migration list from the migration directory specified in config, as well as
+ * the list of completed migrations to check what should be run.
+ */
+export function listAllAndCompleted(config: ISeederConfig, trxOrKnex) {
+  return Promise.all([
+    listAll(config.migrationSource, config.loadExtensions),
+    listCompleted(config.tableName, config.schemaName, trxOrKnex),
+  ]);
+}
+
+/**
+ *
+ * @param migrationSource
+ * @param all
+ * @param completed
+ * @returns
+ */
+export function getNewMigrations(
+  migrationSource: FsMigrations,
+  all: MigrateItem[],
+  completed: string[]
+): MigrateItem[] {
+  return differenceWith(all, completed, (allMigration, completedMigration) => {
+    return (
+      completedMigration === migrationSource.getMigrationName(allMigration)
+    );
+  });
+}
+
+function startsWithNumber(str) {
+  return /^\d/.test(str);
+}
+/**
+ *
+ * @param {FsMigrations} migrationSource -
+ * @param {MigrateItem[]} migrations -
+ * @param {string[]} loadExtensions -
+ * @returns
+ */
+export function filterMigrations(
+  migrationSource: FsMigrations,
+  migrations: MigrateItem[],
+  loadExtensions: string[]
+) {
+  return migrations.filter((migration) => {
+    const migrationName = migrationSource.getMigrationName(migration);
+    const extension = path.extname(migrationName);
+
+    return (
+      loadExtensions.includes(extension) && startsWithNumber(migrationName)
+    );
+  });
+}

packages/server/src/lib/Seeder/SeedMigration.ts

@@ -0,0 +1,222 @@
+import { Knex } from 'knex';
+import Bluebird from 'bluebird';
+import { getTable, getTableName, getLockTableName } from './TableUtils';
+import getMergedConfig from './SeederConfig';
+import {
+  listAllAndCompleted,
+  getNewMigrations,
+  listCompleted,
+  ensureMigrationTables,
+} from './MigrateUtils';
+import { MigrateItem, SeedMigrationContext, ISeederConfig } from './interfaces';
+import { FsMigrations } from './FsMigrations';
+
+export class SeedMigration {
+  knex: Knex;
+  config: ISeederConfig;
+  migrationSource: FsMigrations;
+  context: SeedMigrationContext;
+
+  /**
+   * Constructor method.
+   * @param {Knex} knex - Knex instance.
+   * @param {SeedMigrationContext} context -
+   */
+  constructor(knex: Knex, context: SeedMigrationContext) {
+    this.knex = knex;
+    this.config = getMergedConfig(this.knex.client.config.seeds, undefined);
+    this.migrationSource = this.config.migrationSource;
+    this.context = context;
+  }
+
+  /**
+   * Latest migration.
+   * @returns {Promise<void>}
+   */
+  async latest(config = null): Promise<void> {
+    // Merges the configuration.
+    this.config = getMergedConfig(config, this.config);
+
+    // Ensure migration tables.
+    await ensureMigrationTables(this.config.tableName, null, this.knex);
+
+    // Retrieve all and completed migrations.
+    const [all, completed] = await listAllAndCompleted(this.config, this.knex);
+
+    // Retrieve the new migrations.
+    const migrations = getNewMigrations(this.migrationSource, all, completed);
+
+    // Run the latest migration on one batch.
+    return this.knex.transaction((trx: Knex.Transaction) => {
+      return this.runBatch(migrations, 'up', trx);
+    });
+  }
+
+  /**
+   * Add migration lock flag.
+   * @param {Knex.Transaction} trx
+   * @returns
+   */
+  private migrateLockTable(trx: Knex.Transaction) {
+    const tableName = getLockTableName(this.config.tableName);
+    return getTable(this.knex, tableName, this.config.schemaName)
+      .transacting(trx)
+      .where('is_locked', '=', 0)
+      .update({ is_locked: 1 })
+      .then((rowCount) => {
+        if (rowCount != 1) {
+          throw new Error('Migration table is already locked');
+        }
+      });
+  }
+
+  /**
+   * Add migration lock flag.
+   * @param {Knex.Transaction} trx
+   * @returns
+   */
+  private migrationLock(trx: Knex.Transaction) {
+    return this.migrateLockTable(trx);
+  }
+
+  /**
+   * Free the migration lock flag.
+   * @param {Knex.Transaction} trx
+   * @returns
+   */
+  private freeLock(trx = this.knex): Promise<void> {
+    const tableName = getLockTableName(this.config.tableName);
+
+    return getTable(trx, tableName, this.config.schemaName).update({
+      is_locked: 0,
+    });
+  }
+
+  /**
+   * Returns the latest batch number.
+   * @param trx
+   * @returns
+   */
+  private latestBatchNumber(trx = this.knex): number {
+    return trx
+      .from(getTableName(this.config.tableName, this.config.schemaName))
+      .max('batch as max_batch')
+      .then((obj) => obj[0].max_batch || 0);
+  }
+
+  /**
+   * Runs a batch of `migrations` in a specified `direction`, saving the
+   * appropriate database information as the migrations are run.
+   * @param   {number} batchNo
+   * @param   {MigrateItem[]} migrations
+   * @param   {string} direction
+   * @param   {Knex.Transaction} trx
+   * @returns {Promise<void>}
+   */
+  private waterfallBatch(
+    batchNo: number,
+    migrations: MigrateItem[],
+    direction: string,
+    trx: Knex.Transaction
+  ): Promise<void> {
+    const { tableName } = this.config;
+
+    return Bluebird.each(migrations, (migration) => {
+      const name = this.migrationSource.getMigrationName(migration);
+
+      return this.migrationSource
+        .getMigration(migration)
+        .then((migrationContent) =>
+          this.runMigrationContent(migrationContent.default, direction, trx)
+        )
+        .then(() => {
+          if (direction === 'up') {
+            return trx.into(getTableName(tableName)).insert({
+              name,
+              batch: batchNo,
+              migration_time: new Date(),
+            });
+          }
+          if (direction === 'down') {
+            return trx.from(getTableName(tableName)).where({ name }).del();
+          }
+        });
+    });
+  }
+
+  /**
+   * Runs and builds the given migration class.
+   */
+  private runMigrationContent(Migration, direction, trx) {
+    const instance = new Migration(trx);
+
+    if (this.context.i18n) {
+      instance.setI18n(this.context.i18n);
+    }
+    instance.setTenant(this.context.tenant);
+
+    return instance[direction](trx);
+  }
+
+  /**
+   * Validates some migrations by requiring and checking for an `up` and `down`function.
+   * @param {MigrateItem} migration
+   * @returns {MigrateItem}
+   */
+  async validateMigrationStructure(migration: MigrateItem): MigrateItem {
+    const migrationName = this.migrationSource.getMigrationName(migration);
+
+    // maybe promise
+    const migrationContent = await this.migrationSource.getMigration(migration);
+    if (
+      typeof migrationContent.up !== 'function' ||
+      typeof migrationContent.down !== 'function'
+    ) {
+      throw new Error(
+        `Invalid migration: ${migrationName} must have both an up and down function`
+      );
+    }
+    return migration;
+  }
+
+  /**
+   * Run a batch of current migrations, in sequence.
+   * @param   {MigrateItem[]} migrations
+   * @param   {string} direction
+   * @param   {Knex.Transaction} trx
+   * @returns {Promise<void>}
+   */
+  private async runBatch(
+    migrations: MigrateItem[],
+    direction: string,
+    trx: Knex.Transaction
+  ): Promise<void> {
+    // Adds flag to migration lock.
+    await this.migrationLock(trx);
+
+    // When there is a wrapping transaction, some migrations
+    // could have been done while waiting for the lock:
+    const completed = await listCompleted(
+      this.config.tableName,
+      this.config.schemaName,
+      trx
+    );
+    // Differentiate between all and completed to get new migrations.
+    const newMigrations = getNewMigrations(
+      this.config.migrationSource,
+      migrations,
+      completed
+    );
+    // Retrieve the latest batch number.
+    const batchNo = await this.latestBatchNumber(trx);
+
+    // Increment the next batch number.
+    const newBatchNo = direction === 'up' ? batchNo + 1 : batchNo;
+
+    // Run all migration files in waterfall.
+    await this.waterfallBatch(newBatchNo, newMigrations, direction, trx);
+
+    // Free the migration lock flag.
+    await this.freeLock(trx);
+  }
+}

packages/server/src/lib/Seeder/Seeder.ts

@@ -0,0 +1,11 @@
+
+export class Seeder {
+  knex: any;
+
+  constructor(knex) {
+    this.knex = knex;
+  }
+  up(knex) {}
+  down(knex) {}
+}
+

packages/server/src/lib/Seeder/SeederConfig.ts

@@ -0,0 +1,44 @@
+import { DEFAULT_LOAD_EXTENSIONS, FsMigrations } from './FsMigrations';
+
+const CONFIG_DEFAULT = Object.freeze({
+  extension: 'js',
+  loadExtensions: DEFAULT_LOAD_EXTENSIONS,
+  tableName: 'knex_migrations',
+  schemaName: null,
+  directory: './migrations',
+  disableTransactions: false,
+  disableMigrationsListValidation: false,
+  sortDirsSeparately: false,
+});
+
+export default function getMergedConfig(config, currentConfig) {
+  // config is the user specified config, mergedConfig has defaults and current config
+  // applied to it.
+  const mergedConfig = {
+    ...CONFIG_DEFAULT,
+    ...(currentConfig || {}),
+    ...config,
+  };
+
+  if (
+    config &&
+    // If user specifies any FS related config,
+    // clear specified migrationSource to avoid ambiguity
+    (config.directory ||
+      config.sortDirsSeparately !== undefined ||
+      config.loadExtensions)
+  ) {
+    mergedConfig.migrationSource = null;
+  }
+
+  // If the user has not specified any configs, we need to
+  // default to fs migrations to maintain compatibility
+  if (!mergedConfig.migrationSource) {
+    mergedConfig.migrationSource = new FsMigrations(
+      mergedConfig.directory,
+      mergedConfig.sortDirsSeparately,
+      mergedConfig.loadExtensions
+    );
+  }
+  return mergedConfig;
+}

packages/server/src/lib/Seeder/TableUtils.ts

@@ -0,0 +1,43 @@
+/**
+ * Get schema-aware query builder for a given table and schema name.
+ * @param {Knex} trxOrKnex -
+ * @param {string} tableName -
+ * @param {string} schemaName -
+ * @returns {string}
+ */
+export function getTable(trx, tableName: string, schemaName = null) {
+  return schemaName ? trx(tableName).withSchema(schemaName) : trx(tableName);
+}
+
+/**
+ * Get schema-aware table name.
+ * @param {string} tableName -
+ * @returns {string}
+ */
+export function getTableName(tableName: string, schemaName = null): string {
+  return schemaName ? `${schemaName}.${tableName}` : tableName;
+}
+
+/**
+ * Retrieve the lock table name from given migration table name.
+ * @param {string} tableName
+ * @returns {string}
+ */
+export function getLockTableName(tableName: string): string {
+  return `${tableName}_lock`;
+}
+
+/**
+ * Retireve the lock table name from ginve migration table name with schema.
+ * @param {string} tableName
+ * @param {string} schemaName
+ * @returns {string}
+ */
+export function getLockTableNameWithSchema(
+  tableName: string,
+  schemaName = null
+): string {
+  return schemaName
+    ? `${schemaName} + ${getLockTableName(tableName)}`
+    : getLockTableName(tableName);
+}

packages/server/src/lib/Seeder/TenantSeeder.ts

@@ -0,0 +1,25 @@
+import { Seeder } from "./Seeder";
+
+export class TenantSeeder extends Seeder{
+  public knex: any;
+  public i18n: i18nAPI;
+  public models: any;
+  public tenant: any;
+
+  constructor(knex) {
+    super(knex);
+    this.knex = knex;
+  }
+
+  setI18n(i18n) {
+    this.i18n = i18n;
+  }
+
+  setModels(models) {
+    this.models = models;
+  }
+
+  setTenant(tenant) {
+    this.tenant = tenant;
+  }
+}

packages/server/src/lib/Seeder/Utils.ts

@@ -0,0 +1,42 @@
+import fs from 'fs';
+
+const { promisify } = require('util');
+const readFile = promisify(fs.readFile);
+
+/**
+ * Detarmines the module type of the given file path.
+ * @param {string} filepath
+ * @returns {boolean}
+ */
+async function isModuleType(filepath: string): boolean {
+  if (process.env.npm_package_json) {
+    // npm >= 7.0.0
+    const packageJson = JSON.parse(
+      await readFile(process.env.npm_package_json, 'utf-8')
+    );
+    if (packageJson.type === 'module') {
+      return true;
+    }
+  }
+  return process.env.npm_package_type === 'module' || filepath.endsWith('.mjs');
+}
+
+/**
+ * Imports content of the given file path.
+ * @param {string} filepath
+ * @returns
+ */
+export async function importFile(filepath: string): any {
+  return (await isModuleType(filepath))
+    ? import(require('url').pathToFileURL(filepath))
+    : require(filepath);
+}
+
+/**
+ *
+ * @param {string} moduleName
+ * @returns
+ */
+export async function importWebpackSeedModule(moduleName: string): any {
+  return import(`@/database/seeds/core/${moduleName}`);
+}

packages/server/src/lib/Seeder/constants.ts

@@ -0,0 +1,12 @@
+// Default load extensions.
+export const DEFAULT_LOAD_EXTENSIONS = [
+  '.co',
+  '.coffee',
+  '.eg',
+  '.iced',
+  '.js',
+  '.cjs',
+  '.litcoffee',
+  '.ls',
+  '.ts',
+];

packages/server/src/lib/Seeder/interfaces.ts

@@ -0,0 +1,20 @@
+import { ITenant } from "interfaces";
+
+export interface FsMigrations {}
+
+export interface ISeederConfig {
+  tableName: string;
+  migrationSource: FsMigrations;
+  schemaName?: string;
+  loadExtensions: string[];
+}
+
+export interface MigrateItem {
+  file: string;
+  directory: string;
+}
+
+export interface SeedMigrationContext {
+  i18n: i18nAPI;
+  tenant: ITenant;
+}