alc-english/pimsleur-study/transcribe.mjs

/**
 * Transcribe Pimsleur units via Soniox async STT, preserving the long pauses
 * (the "your turn" gaps that define the Pimsleur method).
 *
 * Reuses the Soniox flow from reactor-module-tools/module-convert/transcribe-soniox.js
 * but with Spanish+English hints and pause annotation, for studying the tape format.
 *
 * Usage:
 *   node transcribe.mjs "<file.mp3>" ["<file2.mp3>" ...]
 *   node transcribe.mjs --level I --units 1,2,15      # convenience for Spanish <level>
 *
 * Output (in ./transcripts/, mirroring the source name):
 *   <name>.txt   — readable transcript: [mm:ss] lines + "(pause Ns)" markers
 *   <name>.json  — raw Soniox tokens + assembled words (for deeper analysis)
 *
 * Env: SONIOX_API_KEY (loaded from reactor-module-tools/.env).
 */

import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const PIMSLEUR_ROOT = "/home/j/projects/Pimsleur Spanish";
const OUT_DIR = path.join(__dirname, "transcripts");

// Load the Soniox key from reactor-module-tools/.env (no dotenv dep needed).
function loadKey() {
  if (process.env.SONIOX_API_KEY) return process.env.SONIOX_API_KEY;
  const envPath = "/home/j/projects/reactor-module-tools/.env";
  try {
    for (const line of fs.readFileSync(envPath, "utf8").split("\n")) {
      const m = /^\s*SONIOX_API_KEY\s*=\s*(.+?)\s*$/.exec(line);
      if (m) return m[1].replace(/^["']|["']$/g, "");
    }
  } catch {}
  return null;
}

const API = "https://api.soniox.com";
const MODEL = "stt-async-v4";
const POLL_INTERVAL_MS = 4000;
const POLL_MAX = 300; // ~20 min ceiling per unit (units are ~30 min audio)
const GAP_NEWLINE_S = 1.2; // new utterance line
const GAP_PAUSE_S = 2.5; // annotate as a "your turn" pause

const authHeaders = (key) => ({ Authorization: `Bearer ${key}` });

async function sx(key, urlPath, opts = {}) {
  const resp = await fetch(`${API}${urlPath}`, { ...opts, headers: { ...authHeaders(key), ...(opts.headers || {}) } });
  const text = await resp.text();
  let json;
  try { json = text ? JSON.parse(text) : {}; } catch { json = { _raw: text }; }
  if (!resp.ok) throw new Error(`${urlPath} -> HTTP ${resp.status}: ${text.slice(0, 300)}`);
  return json;
}

async function uploadFile(key, filePath) {
  const buf = fs.readFileSync(filePath);
  const form = new FormData();
  form.append("file", new Blob([buf]), path.basename(filePath));
  const json = await sx(key, "/v1/files", { method: "POST", body: form });
  return json.id;
}

async function createTranscription(key, fileId) {
  const json = await sx(key, "/v1/transcriptions", {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    // Spanish + English: Pimsleur interleaves an English narrator with Spanish.
    body: JSON.stringify({ model: MODEL, file_id: fileId, language_hints: ["es", "en"] }),
  });
  return json.id;
}

async function waitFor(key, id) {
  for (let i = 0; i < POLL_MAX; i++) {
    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
    const j = await sx(key, `/v1/transcriptions/${id}`);
    if (j.status === "completed") return;
    if (j.status === "error") throw new Error(`soniox: ${j.error_type || ""} ${j.error_message || ""}`.trim());
  }
  throw new Error("soniox: timed out");
}

const getTokens = async (key, id) => (await sx(key, `/v1/transcriptions/${id}/transcript`)).tokens || [];

async function cleanup(key, fileId, tid) {
  for (const p of [`/v1/transcriptions/${tid}`, `/v1/files/${fileId}`]) {
    if (!p.endsWith("/undefined")) { try { await sx(key, p, { method: "DELETE" }); } catch {} }
  }
}

// Soniox returns subword tokens (leading space = new word) with per-token
// language. Reassemble into words, carrying language + timing (seconds).
function tokensToWords(tokens) {
  const words = [];
  for (const t of tokens) {
    const tx = t.text ?? "";
    if (!tx.trim()) continue;
    const startsWord = tx.startsWith(" ") || words.length === 0;
    if (startsWord) {
      words.push({ raw: tx.trim(), start: t.start_ms / 1000, end: t.end_ms / 1000, lang: t.language || null });
    } else {
      const w = words[words.length - 1];
      w.raw += tx;
      w.end = t.end_ms / 1000;
    }
  }
  return words;
}

const mmss = (s) => `${String(Math.floor(s / 60)).padStart(2, "0")}:${String(Math.floor(s % 60)).padStart(2, "0")}`;

// Group words into utterances (gap-split), tagging language and inter-utterance
// pauses so the prompt -> (your turn) -> confirmation rhythm is visible.
function render(words) {
  const lines = [];
  let cur = [];
  const flush = () => {
    if (!cur.length) return;
    const text = cur.map((w) => w.raw).join(" ");
    const langs = [...new Set(cur.map((w) => w.lang).filter(Boolean))];
    const tag = langs.length === 1 ? langs[0].toUpperCase() : langs.length ? "ES/EN" : "??";
    lines.push(`[${mmss(cur[0].start)}] (${tag}) ${text}`);
    cur = [];
  };
  for (let i = 0; i < words.length; i++) {
    cur.push(words[i]);
    const next = words[i + 1];
    const gap = next ? next.start - words[i].end : 0;
    if (!next || gap > GAP_NEWLINE_S) {
      const endAt = words[i].end;
      flush();
      if (next && gap > GAP_PAUSE_S) lines.push(`           … (pause ${gap.toFixed(1)}s) …`);
    }
  }
  return lines.join("\n") + "\n";
}

async function transcribe(key, mp3Path) {
  const rel = path.relative(PIMSLEUR_ROOT, mp3Path);
  const safe = rel.replace(/[\\/]/g, "__").replace(/\.mp3$/i, "");
  const outTxt = path.join(OUT_DIR, `${safe}.txt`);
  const outJson = path.join(OUT_DIR, `${safe}.json`);
  if (fs.existsSync(outTxt)) { console.log(`  skip (done): ${rel}`); return; }

  let fileId, tid;
  const t0 = Date.now();
  try {
    fileId = await uploadFile(key, mp3Path);
    tid = await createTranscription(key, fileId);
    await waitFor(key, tid);
    const tokens = await getTokens(key, tid);
    const words = tokensToWords(tokens);
    fs.mkdirSync(OUT_DIR, { recursive: true });
    fs.writeFileSync(outTxt, render(words));
    fs.writeFileSync(outJson, JSON.stringify({ source: rel, model: MODEL, words }, null, 1));
    console.log(`  ${rel}: ${words.length} words [${((Date.now() - t0) / 1000).toFixed(0)}s] -> ${path.basename(outTxt)}`);
  } finally {
    if (fileId) await cleanup(key, fileId, tid);
  }
}

function resolveArgs() {
  const args = process.argv.slice(2);
  const li = args.indexOf("--level");
  if (li !== -1) {
    const level = args[li + 1];
    const ui = args.indexOf("--units");
    const dir = path.join(PIMSLEUR_ROOT, `Pimsleur - Spanish ${level}`);
    let files = fs.readdirSync(dir).filter((f) => /Unit \d+\.mp3$/i.test(f)).sort();
    if (ui !== -1 && args[ui + 1]) {
      const want = new Set(args[ui + 1].split(",").map((n) => n.trim().padStart(2, "0")));
      files = files.filter((f) => want.has(f.match(/Unit (\d+)/i)[1].padStart(2, "0")));
    }
    return files.map((f) => path.join(dir, f));
  }
  return args.filter((a) => a.endsWith(".mp3"));
}

async function main() {
  const key = loadKey();
  if (!key) { console.error("SONIOX_API_KEY not found"); process.exit(1); }
  const files = resolveArgs();
  if (!files.length) {
    console.error('Usage: node transcribe.mjs "<file.mp3>" ... | --level I --units 1,2,15');
    process.exit(1);
  }
  console.log(`Transcribing ${files.length} file(s) via Soniox ${MODEL}…`);
  for (const f of files) {
    try { await transcribe(key, f); }
    catch (e) { console.error(`  FAILED ${f}: ${e.message}`); }
  }
  console.log("Done.");
}

main().catch((e) => { console.error(e); process.exit(1); });