Merge branch 'esm' of github.com:gchq/CyberChef into node-lib

src/core/ChefWorker.js

@@ -104,12 +104,16 @@ async function bake(data) {
 
         self.postMessage({
             action: "bakeComplete",
-            data: response
+            data: Object.assign(response, {
+                id: data.id
+            })
         });
     } catch (err) {
         self.postMessage({
             action: "bakeError",
-            data: err
+            data: Object.assign(err, {
+                id: data.id
+            })
         });
     }
 }

src/core/lib/Magic.mjs

@@ -229,17 +229,22 @@ class Magic {
         const testEnc = async op => {
             for (let i = 0; i < encodings.length; i++) {
                 const conf = {
-                        op: op,
+                    op: op,
-                        args: [encodings[i]]
+                    args: [encodings[i]]
-                    },
+                };
-                    data = await this._runRecipe([conf], sample.buffer);
 
-                // Only add to the results if it changed the data
+                try {
-                if (!_buffersEqual(data, sample.buffer)) {
+                    const data = await this._runRecipe([conf], sample.buffer);
-                    results.push({
+
-                        data: data,
+                    // Only add to the results if it changed the data
-                        conf: conf
+                    if (!_buffersEqual(data, sample.buffer)) {
-                    });
+                        results.push({
+                            data: data,
+                            conf: conf
+                        });
+                    }
+                } catch (err) {
+                    continue;
                 }
             }
         };
@@ -344,6 +349,11 @@ class Magic {
             aScore += a.entropy;
             bScore += b.entropy;
 
+            // A result with no recipe but matching ops suggests there are better options
+            if ((!a.recipe.length && a.matchingOps.length) &&
+                b.recipe.length)
+                return 1;
+
             return aScore - bScore;
         });
     }
@@ -356,8 +366,9 @@ class Magic {
      * @returns {ArrayBuffer}
      */
     async _runRecipe(recipeConfig, input=this.inputBuffer) {
+        input = input instanceof ArrayBuffer ? input : input.buffer;
         const dish = new Dish();
-        dish.set(input.buffer, Dish.ARRAY_BUFFER);
+        dish.set(input, Dish.ARRAY_BUFFER);
 
         if (ENVIRONMENT_IS_WORKER()) self.loadRequiredModules(recipeConfig);
 

src/core/operations/Magic.mjs

@@ -25,7 +25,8 @@ class Magic extends Operation {
         this.module = "Default";
         this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.";
         this.inputType = "ArrayBuffer";
-        this.outputType = "html";
+        this.outputType = "JSON";
+        this.presentType = "html";
         this.args = [
             {
                 "name": "Depth",
@@ -56,10 +57,25 @@ class Magic extends Operation {
         const ings = state.opList[state.progress].ingValues,
             [depth, intensive, extLang] = ings,
             dish = state.dish,
-            currentRecipeConfig = state.opList.map(op => op.config),
             magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)),
             options = await magic.speculativeExecution(depth, extLang, intensive);
 
+        // Record the current state for use when presenting
+        this.state = state;
+
+        dish.set(options, Dish.JSON);
+        return state;
+    }
+
+    /**
+     * Displays Magic results in HTML for web apps.
+     *
+     * @param {JSON} options
+     * @returns {html}
+     */
+    present(options) {
+        const currentRecipeConfig = this.state.opList.map(op => op.config);
+
         let output = `<table
                 class='table table-hover table-condensed table-bordered'
                 style='table-layout: fixed;'>
@@ -84,9 +100,9 @@ class Magic extends Operation {
         options.forEach(option => {
             // Construct recipe URL
             // Replace this Magic op with the generated recipe
-            const recipeConfig = currentRecipeConfig.slice(0, state.progress)
+            const recipeConfig = currentRecipeConfig.slice(0, this.state.progress)
                     .concat(option.recipe)
-                    .concat(currentRecipeConfig.slice(state.progress + 1)),
+                    .concat(currentRecipeConfig.slice(this.state.progress + 1)),
                 recipeURL = "recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(recipeConfig));
 
             let language = "",
@@ -131,8 +147,8 @@ class Magic extends Operation {
         if (!options.length) {
             output = "Nothing of interest could be detected about the input data.\nHave you tried modifying the operation arguments?";
         }
-        dish.set(output, Dish.HTML);
+
-        return state;
+        return output;
     }
 
 }

src/web/BackgroundWorkerWaiter.mjs

@@ -0,0 +1,156 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import ChefWorker from "worker-loader?inline&fallback=false!../core/ChefWorker";
+
+/**
+ * Waiter to handle conversations with a ChefWorker in the background.
+ */
+class BackgroundWorkerWaiter {
+
+    /**
+     * BackgroundWorkerWaiter constructor.
+     *
+     * @param {App} app - The main view object for CyberChef.
+     * @param {Manager} manager - The CyberChef event manager.
+     */
+    constructor(app, manager) {
+        this.app = app;
+        this.manager = manager;
+
+        this.callbacks = {};
+        this.callbackID = 0;
+        this.completedCallback = -1;
+        this.timeout = null;
+    }
+
+
+    /**
+     * Sets up the ChefWorker and associated listeners.
+     */
+    registerChefWorker() {
+        log.debug("Registering new background ChefWorker");
+        this.chefWorker = new ChefWorker();
+        this.chefWorker.addEventListener("message", this.handleChefMessage.bind(this));
+
+        let docURL = document.location.href.split(/[#?]/)[0];
+        const index = docURL.lastIndexOf("/");
+        if (index > 0) {
+            docURL = docURL.substring(0, index);
+        }
+        this.chefWorker.postMessage({"action": "docURL", "data": docURL});
+    }
+
+
+    /**
+     * Handler for messages sent back by the ChefWorker.
+     *
+     * @param {MessageEvent} e
+     */
+    handleChefMessage(e) {
+        const r = e.data;
+        log.debug("Receiving '" + r.action + "' from ChefWorker in the background");
+
+        switch (r.action) {
+            case "bakeComplete":
+            case "bakeError":
+                if (typeof r.data.id !== "undefined") {
+                    clearTimeout(this.timeout);
+                    this.callbacks[r.data.id].bind(this)(r.data);
+                    this.completedCallback = r.data.id;
+                }
+                break;
+            case "workerLoaded":
+                log.debug("Background ChefWorker loaded");
+                break;
+            case "optionUpdate":
+                // Ignore these messages
+                break;
+            default:
+                log.error("Unrecognised message from background ChefWorker", e);
+                break;
+        }
+    }
+
+
+    /**
+     * Cancels the current bake by terminating the ChefWorker and creating a new one.
+     */
+    cancelBake() {
+        if (this.chefWorker)
+            this.chefWorker.terminate();
+        this.registerChefWorker();
+    }
+
+
+    /**
+     * Asks the ChefWorker to bake the input using the specified recipe.
+     *
+     * @param {string} input
+     * @param {Object[]} recipeConfig
+     * @param {Object} options
+     * @param {number} progress
+     * @param {boolean} step
+     * @param {Function} callback
+     */
+    bake(input, recipeConfig, options, progress, step, callback) {
+        const id = this.callbackID++;
+        this.callbacks[id] = callback;
+
+        this.chefWorker.postMessage({
+            action: "bake",
+            data: {
+                input: input,
+                recipeConfig: recipeConfig,
+                options: options,
+                progress: progress,
+                step: step,
+                id: id
+            }
+        });
+    }
+
+
+    /**
+     * Asks the Magic operation what it can do with the input data.
+     *
+     * @param {string|ArrayBuffer} input
+     */
+    magic(input) {
+        // If we're still working on the previous bake, cancel it before stating a new one.
+        if (this.completedCallback + 1 < this.callbackID) {
+            clearTimeout(this.timeout);
+            this.cancelBake();
+        }
+
+        this.bake(input, [
+            {
+                "op": "Magic",
+                "args": [3, false, false]
+            }
+        ], {}, 0, false, this.magicComplete);
+
+        // Cancel this bake if it takes too long.
+        this.timeout = setTimeout(this.cancelBake.bind(this), 3000);
+    }
+
+
+    /**
+     * Handler for completed Magic bakes.
+     *
+     * @param {Object} response
+     */
+    magicComplete(response) {
+        log.debug("--- Background Magic Bake complete ---");
+        if (!response || response.error) return;
+
+        this.manager.output.backgroundMagicResult(response.dish.value);
+    }
+
+}
+
+
+export default BackgroundWorkerWaiter;

src/web/Manager.mjs

@@ -15,6 +15,7 @@ import OptionsWaiter from "./OptionsWaiter";
 import HighlighterWaiter from "./HighlighterWaiter";
 import SeasonalWaiter from "./SeasonalWaiter";
 import BindingsWaiter from "./BindingsWaiter";
+import BackgroundWorkerWaiter from "./BackgroundWorkerWaiter";
 
 
 /**
@@ -68,6 +69,7 @@ class Manager {
         this.highlighter = new HighlighterWaiter(this.app, this);
         this.seasonal    = new SeasonalWaiter(this.app, this);
         this.bindings    = new BindingsWaiter(this.app, this);
+        this.background  = new BackgroundWorkerWaiter(this.app, this);
 
         // Object to store dynamic handlers to fire on elements that may not exist yet
         this.dynamicHandlers = {};
@@ -84,6 +86,7 @@ class Manager {
         this.recipe.initialiseOperationDragNDrop();
         this.controls.autoBakeChange();
         this.bindings.updateKeybList();
+        this.background.registerChefWorker();
         this.seasonal.load();
     }
 

src/web/OutputWaiter.mjs

@@ -117,6 +117,7 @@ class OutputWaiter {
 
         this.manager.highlighter.removeHighlights();
         this.setOutputInfo(length, lines, duration);
+        this.backgroundMagic();
     }
 
 
@@ -444,6 +445,42 @@ class OutputWaiter {
         return this.dishBuffer;
     }
 
+
+    /**
+     * Triggers the BackgroundWorker to attempt Magic on the current output.
+     */
+    backgroundMagic() {
+        const sample = this.dishStr ? this.dishStr.slice(0, 1000) :
+            this.dishBuffer ? this.dishBuffer.slice(0, 1000) : "";
+
+        if (sample.length) {
+            this.manager.background.magic(sample);
+        }
+    }
+
+
+    /**
+     * Handles the results of a background Magic call.
+     *
+     * @param {Object[]} options
+     */
+    backgroundMagicResult(options) {
+        if (!options.length ||
+            !options[0].recipe.length)
+            return;
+
+        //console.log(options);
+
+        const currentRecipeConfig = this.app.getRecipeConfig();
+        const newRecipeConfig = currentRecipeConfig.concat(options[0].recipe);
+        const recipeURL = "#recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(newRecipeConfig));
+        const opSequence = options[0].recipe.map(o => o.op).join(", ");
+
+        log.log(`Running <a href="${recipeURL}">${opSequence}</a> will result in "${Utils.truncate(options[0].data, 20)}"`);
+        //this.app.setRecipeConfig(newRecipeConfig);
+        //this.app.autoBake();
+    }
+
 }
 
 export default OutputWaiter;