Experiment with hallucination reduction

- update cursorignore
- add hallucination filter training & inference code
- put logging into a central module
- segment metadata logging occurs before filtering
- segment metadata logging is on by default
- check in embedded python setup script
- include trained hallucination filter model

3 files changed, 111 insertions, 2 deletions

diff --git a/ui/build_scripts/setup-embedded-python.js b/ui/build_scripts/setup-embedded-python.js
new file mode 100644
index 0000000..0622915
--- /dev/null
+++ b/ui/build_scripts/setup-embedded-python.js
@@ -0,0 +1,104 @@
+const { execSync } = require('child_process');
+const path = require('path');
+const fs = require('fs');
+const https = require('https');
+const { promisify } = require('util');
+const stream = require('stream');
+const pipeline = promisify(stream.pipeline);
+const extract = require('extract-zip');
+
+const projectRoot = path.join(__dirname, '..', '..');
+const pythonPath = path.join(projectRoot, 'python_embedded');
+const dllPath = path.join(projectRoot, 'dll_empty');
+
+const PYTHON_URL = 'https://www.python.org/ftp/python/3.10.11/python-3.10.11-embed-amd64.zip';
+const PIP_URL = 'https://bootstrap.pypa.io/get-pip.py';
+
+async function downloadFile(url, dest) {
+    console.log(`Downloading ${url}...`);
+    const file = fs.createWriteStream(dest);
+
+    return new Promise((resolve, reject) => {
+        https.get(url, (response) => {
+            if (response.statusCode === 302 || response.statusCode === 301) {
+                // Handle redirect
+                return downloadFile(response.headers.location, dest).then(resolve).catch(reject);
+            }
+
+            response.pipe(file);
+            file.on('finish', () => {
+                file.close();
+                console.log(`Downloaded to ${dest}`);
+                resolve();
+            });
+        }).on('error', (err) => {
+            fs.unlink(dest, () => {}); // Delete the file on error
+            reject(err);
+        });
+    });
+}
+
+async function setupEmbeddedPython() {
+    console.log('Setting up embedded Python...');
+
+    // Delete existing directories
+    if (fs.existsSync(pythonPath)) {
+        fs.rmSync(pythonPath, { recursive: true, force: true });
+        console.log('Deleted existing Python directory');
+    }
+    if (fs.existsSync(dllPath)) {
+        fs.rmSync(dllPath, { recursive: true, force: true });
+        console.log('Deleted existing dll directory');
+    }
+
+    // Create directories
+    fs.mkdirSync(pythonPath, { recursive: true });
+    fs.mkdirSync(dllPath, { recursive: true });
+    console.log('Created Python and dll directories');
+
+    // Download Python
+    const pythonZip = path.join(projectRoot, 'python-3.10.11-embed-amd64.zip');
+    if (!fs.existsSync(pythonZip)) {
+        await downloadFile(PYTHON_URL, pythonZip);
+    }
+
+    // Extract Python
+    console.log('Extracting Python...');
+    await extract(pythonZip, { dir: pythonPath });
+    console.log('Python extracted successfully');
+
+    // Update python310._pth to include the app directory and enable site packages
+    const pthFile = path.join(pythonPath, 'python310._pth');
+    const pthContent = fs.readFileSync(pthFile, 'utf8');
+    fs.writeFileSync(pthFile, pthContent + '\n../app\nimport site\n');
+    console.log('Updated python310._pth');
+
+    // Download get-pip.py
+    const getPipPath = path.join(pythonPath, 'get-pip.py');
+    await downloadFile(PIP_URL, getPipPath);
+
+    // Install pip
+    console.log('Installing pip...');
+    try {
+        execSync(`"${path.join(pythonPath, 'python.exe')}" "${getPipPath}"`, {
+            stdio: 'inherit',
+            cwd: pythonPath
+        });
+        console.log('pip installed successfully');
+    } catch (error) {
+        console.error('Failed to install pip:', error);
+        process.exit(1);
+    }
+
+    // Clean up
+    fs.unlinkSync(getPipPath);
+
+    console.log('Embedded Python setup complete!');
+}
+
+// Run the setup
+setupEmbeddedPython().catch(err => {
+    console.error('Setup failed:', err);
+    process.exit(1);
+});
+
diff --git a/ui/config-schema.js b/ui/config-schema.js
index fb90f3f..39b74b6 100644
--- a/ui/config-schema.js
+++ b/ui/config-schema.js
@@ -29,7 +29,7 @@ const CONFIG_SCHEMA = {
     enable_debug_mode: { type: 'boolean', default: 0 },
     enable_previews: { type: 'boolean', default: 1 },
     save_audio: { type: 'boolean', default: 0 },
-    enable_segment_logging: { type: 'boolean', default: 0 },
+    enable_segment_logging: { type: 'boolean', default: 1 },
     use_cpu: { type: 'boolean', default: 0 },
     enable_lowercase_filter: { type: 'boolean', default: 0 },
     enable_uppercase_filter: { type: 'boolean', default: 0 },
@@ -54,4 +54,4 @@ if (typeof module !== 'undefined' && module.exports) {
 } else {
     window.CONFIG_SCHEMA = CONFIG_SCHEMA;
     window.getDefaultConfig = getDefaultConfig;
-} 
\ No newline at end of file
+} 
diff --git a/ui/package.json b/ui/package.json
index ce22dee..a3647dc 100644
--- a/ui/package.json
+++ b/ui/package.json
@@ -63,6 +63,11 @@
         "from": "../Sounds",
         "to": "Sounds",
         "filter": ["*.wav"]
+      },
+      {
+        "from": "../Models",
+        "to": "Models",
+        "filter": ["**/*.pkl"]
       }
     ],
     "win": {