independentproof-website/scripts/nextexplorer.mjs

#!/usr/bin/env node
/**
 * Read and write files in NextExplorer, and publish the site to the folder Start9 Pages serves.
 *
 *   node scripts/nextexplorer.mjs <command> [args]      (or: npm run nx -- <command> [args])
 *
 *   whoami                          the account, and the locations it can reach, with the path
 *                                   Start9 Pages needs for each
 *   ls [path]                       list a folder (no path: the account's locations)
 *   get <remote-file> [local-file]  download a file
 *   put <local> <remote-folder>     upload a file, or a folder's contents, into a folder
 *                                   (--replace: overwrite names that already exist)
 *   mkdir <path>                    create a folder, and any missing parents
 *   mv <remote> <new-name>          rename in place
 *   rm <remote> [--permanent]       delete (to NextExplorer's trash, unless --permanent)
 *   deploy [--dry-run] [--no-build] build the site and publish it to NX_DEST
 *   rollback                        swap NX_DEST with the copy the last deploy kept
 *
 * Remote paths start with where the account's access comes from, as `whoami` shows it:
 *   - a location (granted in the account's Volumes tab):       Websites/independentproof/index.html
 *   - a folder shared with the account, by the share's label:  share:Websites/independentproof/index.html
 *     If several shares carry that label, the read-write one is used.
 *
 * SETTINGS come from deploy.local.env (git-ignored) and the environment, the environment winning:
 *   NX_URL         NextExplorer address, e.g. https://<server>.local:<port>
 *   NX_USER        account email
 *   NX_PASS        the password, or NX_PASS_FILE: a file holding only the password
 *                  (default ~/.config/nextexplorer/<account name>.password, which should be mode 600)
 *   NX_CA_FILE     the StartOS server's root CA, so TLS is verified rather than switched off
 *   NX_DEST        the folder the site is published to, e.g. share:Websites/independentproof
 *   NX_PAGES_PATH  the same folder as Start9 Pages names it (from the storage root, e.g.
 *                  Files/Websites/independentproof). Needed for a share, whose location on disk an account
 *                  that doesn't own it can't see; worked out automatically for a location.
 *   SITE_URL       where Start9 Pages serves the site; printed after a deploy
 *
 * WHY DEPLOY SWAPS FOLDERS. NextExplorer never overwrites on upload: a second index.html lands as
 * "index (1).html". So deploy uploads the whole build into a fresh sibling folder, then renames the
 * live folder aside to "<name>.previous" and the new one into place. Stale files can't linger, the
 * live site is never half-updated, and `rollback` can swap the previous copy back.
 *
 * API: read from NextExplorer 3.1.0's backend (backend/src/routes). Login is a session cookie from
 * POST /api/auth/login; uploads are multipart POST /api/upload with `uploadTo` (an existing folder)
 * and `relativePath` (may include new subfolders) sent before the file part, one file per request.
 */

import { execFileSync, spawnSync } from 'node:child_process';
import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from 'node:fs';
import { homedir } from 'node:os';
import { basename, dirname, join, posix, relative, sep } from 'node:path';
import { fileURLToPath } from 'node:url';

const root = join(dirname(fileURLToPath(import.meta.url)), '..');
const dist = join(root, 'dist');

// ---------------------------------------------------------------------------- settings

const localEnv = join(root, 'deploy.local.env');
if (existsSync(localEnv)) {
  for (const line of readFileSync(localEnv, 'utf8').split('\n')) {
    const m = line.match(/^\s*([A-Z_][A-Z0-9_]*)\s*=\s*(.*?)\s*$/);
    if (m && process.env[m[1]] === undefined) process.env[m[1]] = m[2].replace(/^(['"])(.*)\1$/, '$2');
  }
}
const env = process.env;
const expand = (p) => (p ? p.replace(/^~(?=$|\/)/, homedir()) : p);

// Node reads extra CA certificates only at startup, so re-run once with the server's CA added.
const caFile = expand(env.NX_CA_FILE);
if (caFile && env.NODE_EXTRA_CA_CERTS !== caFile) {
  if (!existsSync(caFile)) fail(`NX_CA_FILE not found: ${caFile}`);
  const r = spawnSync(process.execPath, process.argv.slice(1), {
    stdio: 'inherit',
    env: { ...env, NODE_EXTRA_CA_CERTS: caFile },
  });
  process.exit(r.status ?? 1);
}

const BASE = (env.NX_URL ?? '').replace(/\/+$/, '');

function fail(msg) {
  console.error(`error: ${msg}`);
  process.exit(1);
}

function password() {
  if (env.NX_PASS) return env.NX_PASS;
  const account = (env.NX_USER ?? '').split('@')[0];
  const file = expand(env.NX_PASS_FILE) || join(homedir(), '.config/nextexplorer', `${account}.password`);
  if (!existsSync(file)) {
    fail(`no password: set NX_PASS, or put it (and nothing else) in ${file} with mode 600`);
  }
  if ((statSync(file).mode & 0o077) !== 0) console.warn(`! ${file} is readable by others; chmod 600 it`);
  return readFileSync(file, 'utf8').replace(/\r?\n$/, '');
}

// ---------------------------------------------------------------------------- API

let cookie = '';

async function api(method, path, { json, form, raw } = {}) {
  const headers = { cookie };
  let body;
  if (json !== undefined) {
    headers['Content-Type'] = 'application/json';
    body = JSON.stringify(json);
  } else if (form) {
    body = form;
  }
  const res = await fetch(`${BASE}${path}`, { method, headers, body });
  if (raw) {
    if (!res.ok) throw new Error(`${method} ${path}: ${res.status} ${await errorText(res)}`);
    return res;
  }
  const text = await res.text();
  if (!res.ok) throw new Error(`${method} ${path}: ${res.status} ${errorOf(text)}`);
  return text ? JSON.parse(text) : null;
}

const errorOf = (text) => {
  try {
    const j = JSON.parse(text);
    return j.error?.message ?? j.error ?? j.message ?? text;
  } catch {
    return text;
  }
};
const errorText = async (res) => errorOf(await res.text());

/**
 * The session is saved and reused (it lasts 30 days). NextExplorer allows only 10 logins per 15
 * minutes per address, so logging in on every command would lock the account out mid-deploy.
 */
function sessionFile() {
  const account = (env.NX_USER ?? '').split('@')[0];
  return join(homedir(), '.config/nextexplorer', `${account}.session`);
}

async function login() {
  if (!BASE) fail('NX_URL is not set (deploy.local.env)');
  if (!env.NX_USER) fail('NX_USER is not set (deploy.local.env)');

  const saved = sessionFile();
  if (existsSync(saved)) {
    cookie = readFileSync(saved, 'utf8').trim();
    const me = await fetch(`${BASE}/api/auth/me`, { headers: { cookie } });
    if (me.ok) {
      const body = await me.json();
      const user = body.user ?? body;
      if (user?.email === env.NX_USER || user?.username === env.NX_USER.split('@')[0]) return user;
    }
    cookie = '';
  }

  const res = await fetch(`${BASE}/api/auth/login`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ email: env.NX_USER, password: password() }),
  });
  if (!res.ok) fail(`login failed: ${res.status} ${await errorText(res)}`);
  const cookies = res.headers.getSetCookie();
  if (cookies.length === 0) fail('login succeeded but set no session cookie');
  cookie = cookies.map((c) => c.split(';')[0]).join('; ');
  mkdirSync(dirname(saved), { recursive: true, mode: 0o700 });
  writeFileSync(saved, cookie, { mode: 0o600 });
  return (await res.json()).user;
}

const clean = (p) => (p ?? '').replace(/\\/g, '/').replace(/^\/+|\/+$/g, '');

/** Folders shared with this account: [{ label, shareToken, accessMode, ... }]. */
let sharesCache;
async function sharedWithMe() {
  sharesCache ??= (await api('GET', '/api/shares/shared-with-me')).shares ?? [];
  return sharesCache;
}

/** `share:<label>/rest` → NextExplorer's own `share/<token>/rest`. Other paths pass through. */
async function resolve(path) {
  const p = clean(path);
  const m = p.match(/^share:([^/]+)(\/.*)?$/);
  if (!m) return p;
  const matches = (await sharedWithMe()).filter((s) => s.label === m[1]);
  if (!matches.length) throw new Error(`no share labelled "${m[1]}" (see whoami)`);
  const share = matches.find((s) => s.accessMode === 'readwrite') ?? matches[0];
  return clean(`share/${share.shareToken}${m[2] ?? ''}`);
}

/** How many leading segments name the root of access, which an account can't create. */
const rootDepth = (p) => (p.startsWith('share/') ? 2 : 1);
const encodePath = (p) => clean(p).split('/').filter(Boolean).map(encodeURIComponent).join('/');

/** Items in a folder: [{ name, kind ('directory' or an extension), size, dateModified }]. */
async function list(path) {
  const r = await api('GET', `/api/browse/${encodePath(path)}`);
  return r.items ?? r;
}

/** The item at `path`, or null. The parent must exist. */
async function stat(path) {
  const p = clean(path);
  const parent = posix.dirname(p);
  const items = await list(parent === '.' ? '' : parent);
  return items.find((i) => i.name === posix.basename(p)) ?? null;
}

async function exists(path) {
  try {
    return (await stat(path)) !== null;
  } catch {
    return false;
  }
}

async function mkdirp(path) {
  const parts = clean(path).split('/');
  // The first segment(s) name a location or share, which an account can't create.
  for (let i = rootDepth(clean(path)) + 1; i <= parts.length; i++) {
    const p = parts.slice(0, i).join('/');
    if (await exists(p)) continue;
    // NextExplorer picks "name 2" rather than fail when a name is taken, so check what it made.
    const r = await api('POST', '/api/files/folder', { json: { path: posix.dirname(p), name: posix.basename(p) } });
    if (r?.item?.name && r.item.name !== posix.basename(p)) throw new Error(`created "${r.item.name}", not "${p}"`);
  }
}

async function rename(path, newName) {
  const p = clean(path);
  await api('POST', '/api/files/rename', {
    json: { path: posix.dirname(p), name: posix.basename(p), newName },
  });
}

async function remove(path, permanent) {
  const p = clean(path);
  await api('DELETE', '/api/files', {
    json: { items: [{ path: posix.dirname(p), name: posix.basename(p) }], permanent },
  });
}

/**
 * Upload one local file to `<uploadTo>/<relativePath>`. uploadTo must exist; folders inside
 * relativePath are created. Fails rather than accept the "name (1)" a taken name would get.
 */
async function uploadFile(localPath, uploadTo, relativePath) {
  const form = new FormData();
  // Text fields first: the server reads them while the file part is still streaming in.
  form.append('uploadTo', clean(uploadTo));
  form.append('relativePath', clean(relativePath));
  form.append('filedata', new Blob([readFileSync(localPath)]), posix.basename(relativePath));
  const [placed] = await api('POST', '/api/upload', { form });
  const want = posix.basename(relativePath);
  if (placed?.name !== want) {
    throw new Error(`"${relativePath}" was stored as "${placed?.name}" because the name was taken`);
  }
}

/** Run `task` over `items`, `n` at a time. */
async function pool(items, n, task) {
  let next = 0;
  await Promise.all(
    Array.from({ length: Math.min(n, items.length) }, async () => {
      while (next < items.length) await task(items[next++]);
    }),
  );
}

/** Every file under `dir`, relative, with POSIX separators. */
function walk(dir, base = dir) {
  const out = [];
  for (const entry of readdirSync(dir)) {
    const full = join(dir, entry);
    if (statSync(full).isDirectory()) out.push(...walk(full, base));
    else out.push(relative(base, full).split(sep).join('/'));
  }
  return out.sort();
}

/** The path Start9 Pages needs for a NextExplorer path: relative to the storage root, /mnt. */
async function pagesPath(path) {
  if (clean(path).startsWith('share/')) return null; // not visible to an account that doesn't own it
  const [first, ...rest] = clean(path).split('/');
  const vol = (await api('GET', '/api/volumes')).find((v) => v.name === first);
  if (!vol?.actualPath) return null;
  const onDisk = vol.actualPath.replace(/^\/mnt\/?/, '');
  if (onDisk.startsWith('_users')) return null; // a personal space, which Start9 Pages can't be pointed at
  return [onDisk, ...rest].filter(Boolean).join('/');
}

// ---------------------------------------------------------------------------- commands

const [cmd, ...args] = process.argv.slice(2);
const flags = new Set(args.filter((a) => a.startsWith('--')));
const pos = args.filter((a) => !a.startsWith('--'));

const commands = {
  async whoami() {
    const user = await login();
    console.log(`${user.email ?? user.username} (${(user.roles ?? []).join(', ') || 'user'}) at ${BASE}\n`);
    const vols = await api('GET', '/api/volumes');
    const shares = await sharedWithMe();
    if (!vols.length && !shares.length) console.log('No locations or shares. An admin grants a location in the account\'s Volumes tab, or shares a folder with it.');
    if (vols.length) console.log('Locations:');
    for (const v of vols) {
      const pages = await pagesPath(v.name);
      console.log(`  ${v.name.padEnd(28)} Start9 Pages path: ${pages ?? '(personal space; not servable)'}`);
    }
    if (shares.length) console.log('Shared with this account (use as share:<label>/...):');
    for (const sh of shares) {
      console.log(`  share:${sh.label.padEnd(22)} ${sh.accessMode.padEnd(10)} token ${sh.shareToken}${sh.expiresAt ? `, expires ${sh.expiresAt}` : ''}`);
    }
  },

  async ls() {
    await login();
    const path = pos[0] ? await resolve(pos[0]) : '';
    const items = path ? await list(path) : await api('GET', '/api/volumes');
    for (const i of items) {
      const dir = !path || i.kind === 'directory';
      console.log(`${dir ? 'd' : '-'} ${dir ? ''.padStart(10) : String(i.size).padStart(10)}  ${i.name}${dir ? '/' : ''}`);
    }
  },

  async get() {
    const [remote, local = posix.basename(clean(pos[0] ?? ''))] = pos;
    if (!remote) fail('usage: get <remote-file> [local-file]');
    await login();
    const res = await api('POST', '/api/download', { json: { path: await resolve(remote) }, raw: true });
    mkdirSync(dirname(local) || '.', { recursive: true });
    writeFileSync(local, Buffer.from(await res.arrayBuffer()));
    console.log(`${remote} → ${local}`);
  },

  async put() {
    const [local, remoteArg] = pos;
    if (!local || !remoteArg) fail('usage: put <local-file-or-folder> <remote-folder> [--replace]');
    await login();
    const remoteDir = await resolve(remoteArg);
    const isDir = statSync(local).isDirectory();
    const files = isDir ? walk(local) : [basename(local)];
    const from = isDir ? local : dirname(local);
    await mkdirp(remoteDir);
    let n = 0;
    await pool(files, 4, async (f) => {
      const target = `${clean(remoteDir)}/${f}`;
      if (flags.has('--replace') && (await exists(target))) await remove(target, true);
      await uploadFile(join(from, f), remoteDir, f);
      console.log(`  [${String(++n).padStart(3)}/${files.length}] ${target}`);
    });
  },

  async mkdir() {
    if (!pos[0]) fail('usage: mkdir <path>');
    await login();
    await mkdirp(await resolve(pos[0]));
    console.log(`ok: ${clean(pos[0])}`);
  },

  async mv() {
    const [path, newName] = pos;
    if (!path || !newName) fail('usage: mv <remote> <new-name>');
    await login();
    await rename(await resolve(path), newName);
    console.log(`${clean(path)} → ${newName}`);
  },

  async rm() {
    if (!pos[0]) fail('usage: rm <remote> [--permanent]');
    await login();
    await remove(await resolve(pos[0]), flags.has('--permanent'));
    console.log(`deleted${flags.has('--permanent') ? ' permanently' : ' (to trash)'}: ${clean(pos[0])}`);
  },

  async deploy() {
    const destArg = clean(env.NX_DEST);
    if (!destArg.includes('/')) fail('NX_DEST must be a folder inside a location or share, e.g. share:Websites/independentproof');

    if (!flags.has('--no-build')) {
      console.log('building…');
      execFileSync('npm', ['run', 'build'], { cwd: root, stdio: 'inherit' });
    }
    // .htaccess is for Apache hosting; Start9 Pages' nginx would serve it as a plain file.
    const files = walk(dist).filter((f) => !f.split('/').some((s) => s.startsWith('.')));
    if (!files.length) fail('dist/ is empty; nothing to deploy');
    const kb = files.reduce((n, f) => n + statSync(join(dist, f)).size, 0) / 1024;
    console.log(`\n${files.length} files, ${kb.toFixed(0)} KB → ${destArg}`);
    if (flags.has('--dry-run')) {
      files.forEach((f) => console.log(`  ${destArg}/${f}`));
      console.log('\ndry run: nothing was sent');
      return;
    }

    await login();
    const dest = await resolve(destArg);
    const parent = posix.dirname(dest);
    const name = posix.basename(dest);
    if (parent.split('/').length > rootDepth(parent)) await mkdirp(parent);
    const stamp = new Date().toISOString().replace(/\D/g, '').slice(0, 14);
    const staging = `${name}.incoming-${stamp}`;

    let n = 0;
    await pool(files, 4, async (f) => {
      await uploadFile(join(dist, f), parent, `${staging}/${f}`);
      if (++n % 10 === 0 || n === files.length) console.log(`  uploaded ${n}/${files.length}`);
    });

    const previous = `${name}.previous`;
    if (await exists(`${parent}/${previous}`)) await remove(`${parent}/${previous}`, true);
    const hadLive = await exists(dest);
    if (hadLive) await rename(dest, previous);
    await rename(`${parent}/${staging}`, name);
    console.log(`\npublished ${destArg}${hadLive ? ` (the last version is kept as ${previous}; \`rollback\` restores it)` : ''}`);

    const pages = env.NX_PAGES_PATH || (await pagesPath(dest));
    console.log(
      pages
        ? `Start9 Pages path: ${pages}`
        : '! set NX_PAGES_PATH to where this folder sits on disk (e.g. Files/Websites/independentproof) to have it printed',
    );
    if (env.SITE_URL) console.log(`Site: ${env.SITE_URL}`);
  },

  async rollback() {
    await login();
    const dest = await resolve(env.NX_DEST);
    const parent = posix.dirname(dest);
    const name = posix.basename(dest);
    const previous = `${parent}/${name}.previous`;
    if (!(await exists(previous))) fail(`nothing to roll back to: ${previous} doesn't exist`);
    const swap = `${name}.swap-${Date.now()}`;
    await rename(dest, swap);
    await rename(previous, name);
    await rename(`${parent}/${swap}`, `${name}.previous`);
    console.log(`restored ${clean(env.NX_DEST)} from ${name}.previous (the version it replaced is now ${name}.previous)`);
  },
};

if (!commands[cmd]) {
  console.log(readFileSync(fileURLToPath(import.meta.url), 'utf8').match(/\/\*\*([\s\S]*?)\*\//)[1].replace(/^ \* ?/gm, ''));
  process.exit(cmd ? 1 : 0);
}
try {
  await commands[cmd]();
} catch (err) {
  fail(err.cause ? `${err.message}: ${err.cause.code ?? err.cause.message ?? err.cause}` : err.message);
}