Downloadble version is now a .zip file instead of a single .htm file

2024-12-28 05:23:10 +00:00 · 2019-04-12 18:54:31 +01:00 · 2019-04-12 18:54:31 +01:00 · 8f450501cc
commit 8f450501cc
parent b3ae0e577a
9 changed files with 631 additions and 433 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -30,8 +30,9 @@ deploy:
    skip_cleanup: true
    api_key:
      secure: "HV1WSKv4l/0Y2bKKs1iBJocBcmLj08PCRUeEM/jTwA4jqJ8EiLHWiXtER/D5sEg2iibRVKd2OQjfrmS6bo4AiwdeVgAKmv0FtS2Jw+391N8Nd5AkEANHa5Om/IpHLTL2YRAjpJTsDpY72bMUTJIwjQA3TFJkgrpOw6KYfohOcgbxLpZ4XuNJRU3VL4Hsxdv5V9aOVmfFOmMOVPQlakXy7NgtW5POp1f2WJwgcZxylkR1CjwaqMyXmSoVl46pyH3tr5+dptsQoKSGdi6sIHGA60oDotFPcm+0ifa47wZw+vapuuDi4tdNxhrHGaDMG8xiE0WFDHwQUDlk2/+W7j9SEX0H3Em7us371JXRp56EDwEcDa34VpVkC6i8HGcHK55hnxVbMZXGf3qhOFD8wY7qMbjMRvIpucrMHBi86OfkDfv0vDj2LyvIl5APj/AX50BrE0tfH1MZbH26Jkx4NdlkcxQ14GumarmUqfmVvbX/fsoA6oUuAAE9ZgRRi3KHO4wci6KUcRfdm+XOeUkaBFsL86G3EEYIvrtBTuaypdz+Cx7nd1iPZyWMx5Y1gXnVzha4nBdV4+7l9JIsFggD8QVpw2uHXQiS1KXFjOeqA3DBD8tjMB7q26Fl2fD3jkOo4BTbQ2NrRIZUu/iL+fOmMPsyMt2qulB0yaSBCfkbEq8xrUA="
+    file_glob: true
    file:
-      - build/prod/cyberchef.htm
+      - build/prod/*.zip
      - build/node/CyberChef.js
    on:
      repo: gchq/CyberChef
--- a/Gruntfile.js
+++ b/Gruntfile.js
@ -4,7 +4,6 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
 const NodeExternals = require("webpack-node-externals");
-const Inliner = require("web-resource-inliner");
 const glob = require("glob");
 const path = require("path");

@ -43,18 +42,16 @@ module.exports = function (grunt) {

    grunt.registerTask("prod",
        "Creates a production-ready build. Use the --msg flag to add a compile message.",
-        ["eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web", "inline", "chmod"]);
+        [
+            "eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web",
+            "copy:standalone", "zip:standalone", "clean:standalone", "chmod"
+        ]);

    grunt.registerTask("default",
        "Lints the code base",
        ["eslint", "exec:repoSize"]);

-    grunt.registerTask("inline",
-        "Compiles a production build of CyberChef into a single, portable web page.",
-        ["exec:generateConfig", "webpack:webInline", "runInliner", "clean:inlineScripts"]);

-
-    grunt.registerTask("runInliner", runInliner);
    grunt.registerTask("doc", "docs");
    grunt.registerTask("tests", "test");
    grunt.registerTask("lint", "eslint");
@ -72,6 +69,7 @@ module.exports = function (grunt) {
    grunt.loadNpmTasks("grunt-accessibility");
    grunt.loadNpmTasks("grunt-concurrent");
    grunt.loadNpmTasks("grunt-contrib-connect");
+    grunt.loadNpmTasks("grunt-zip");


    // Project configuration
@ -94,32 +92,6 @@ module.exports = function (grunt) {
        },
        moduleEntryPoints = listEntryModules();

-    /**
-     * Compiles a production build of CyberChef into a single, portable web page.
-     */
-    function runInliner() {
-        const done = this.async();
-        Inliner.html({
-            relativeTo: "build/prod/",
-            fileContent: grunt.file.read("build/prod/cyberchef.htm"),
-            images: true,
-            svgs: true,
-            scripts: true,
-            links: true,
-            strict: true
-        }, function(error, result) {
-            if (error) {
-                if (error instanceof Error) {
-                    done(error);
-                } else {
-                    done(new Error(error));
-                }
-            } else {
-                grunt.file.write("build/prod/cyberchef.htm", result);
-                done(true);
-            }
-        });
-    }

    /**
     * Generates an entry list for all the modules.
@ -130,7 +102,7 @@ module.exports = function (grunt) {
        glob.sync("./src/core/config/modules/*.mjs").forEach(file => {
            const basename = path.basename(file);
            if (basename !== "Default.mjs" && basename !== "OpModules.mjs")
-                entryModules[basename.split(".mjs")[0]] = path.resolve(file);
+                entryModules["modules/" + basename.split(".mjs")[0]] = path.resolve(file);
        });

        return entryModules;
@ -143,7 +115,7 @@ module.exports = function (grunt) {
            node: ["build/node/*"],
            config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"],
            docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"],
-            inlineScripts: ["build/prod/scripts.js"],
+            standalone: ["build/prod/CyberChef*.html"]
        },
        eslint: {
            options: {
@ -225,33 +197,6 @@ module.exports = function (grunt) {
                    ]
                };
            },
-            webInline: {
-                mode: "production",
-                target: "web",
-                entry: "./src/web/index.js",
-                output: {
-                    filename: "scripts.js",
-                    path: __dirname + "/build/prod"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(Object.assign({}, BUILD_CONSTANTS, {
-                        INLINE: "true"
-                    })),
-                    new HtmlWebpackPlugin({
-                        filename: "cyberchef.htm",
-                        template: "./src/web/html/index.html",
-                        compileTime: compileTime,
-                        version: pkg.version + "s",
-                        inline: true,
-                        minify: {
-                            removeComments: true,
-                            collapseWhitespace: true,
-                            minifyJS: true,
-                            minifyCSS: true
-                        }
-                    }),
-                ]
-            },
            node: {
                mode: "production",
                target: "node",
@ -317,6 +262,18 @@ module.exports = function (grunt) {
                }
            }
        },
+        zip: {
+            standalone: {
+                cwd: "build/prod/",
+                src: [
+                    "build/prod/**/*",
+                    "!build/prod/index.html",
+                    "!build/prod/BundleAnalyzerReport.html",
+                    "!build/prod/sitemap.js"
+                ],
+                dest: `build/prod/CyberChef_v${pkg.version}.zip`
+            }
+        },
        connect: {
            prod: {
                options: {
@ -329,10 +286,16 @@ module.exports = function (grunt) {
            ghPages: {
                options: {
                    process: function (content, srcpath) {
-                        // Add Google Analytics code to index.html
                        if (srcpath.indexOf("index.html") >= 0) {
+                            // Add Google Analytics code to index.html
                            content = content.replace("</body></html>",
                                grunt.file.read("src/web/static/ga.html") + "</body></html>");
+
+                            // Add Structured Data for SEO
+                            content = content.replace("</head>",
+                                "<script type='application/ld+json'>" +
+                                JSON.stringify(grunt.file.read("src/web/static/structuredData.json")) +
+                                "</script>");
                            return grunt.template.process(content, srcpath);
                        } else {
                            return content;
@ -351,6 +314,28 @@ module.exports = function (grunt) {
                        dest: "build/prod/"
                    },
                ]
+            },
+            standalone: {
+                options: {
+                    process: function (content, srcpath) {
+                        if (srcpath.indexOf("index.html") >= 0) {
+                            // Replace download link with version number
+                            content = content.replace(/<a [^>]+>Download CyberChef.+?<\/a>/,
+                                `<span>Version ${pkg.version}</span>`);
+
+                            return grunt.template.process(content, srcpath);
+                        } else {
+                            return content;
+                        }
+                    },
+                    noProcess: ["**", "!**/*.html"]
+                },
+                files: [
+                    {
+                        src: "build/prod/index.html",
+                        dest: `build/prod/CyberChef_v${pkg.version}.html`
+                    }
+                ]
            }
        },
        chmod: {
@ -406,7 +391,7 @@ module.exports = function (grunt) {
                command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
            },
            browserTests: {
-                command: "./node_modules/.bin/nightwatch --env prod,inline"
+                command: "./node_modules/.bin/nightwatch --env prod"
            }
        },
    });
--- a/nightwatch.json
+++ b/nightwatch.json
@ -23,10 +23,6 @@

    "prod": {
      "launch_url": "http://localhost:8000/index.html"
-    },
-
-    "inline": {
-      "launch_url": "http://localhost:8000/cyberchef.htm"
    }

  }
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@ -30,17 +30,17 @@
  "main": "build/node/CyberChef.js",
  "bugs": "https://github.com/gchq/CyberChef/issues",
  "devDependencies": {
-    "@babel/core": "^7.4.0",
-    "@babel/plugin-transform-runtime": "^7.4.0",
-    "@babel/preset-env": "^7.4.2",
-    "autoprefixer": "^9.5.0",
+    "@babel/core": "^7.4.3",
+    "@babel/plugin-transform-runtime": "^7.4.3",
+    "@babel/preset-env": "^7.4.3",
+    "autoprefixer": "^9.5.1",
    "babel-eslint": "^10.0.1",
    "babel-loader": "^8.0.5",
    "babel-plugin-syntax-dynamic-import": "^6.18.0",
    "chromedriver": "^2.46.0",
    "colors": "^1.3.3",
    "css-loader": "^2.1.1",
-    "eslint": "^5.15.3",
+    "eslint": "^5.16.0",
    "exports-loader": "^0.7.0",
    "file-loader": "^3.0.1",
    "grunt": "^1.0.4",
@ -53,13 +53,14 @@
    "grunt-contrib-watch": "^1.1.0",
    "grunt-eslint": "^21.0.0",
    "grunt-exec": "~3.0.0",
-    "grunt-jsdoc": "^2.3.0",
+    "grunt-jsdoc": "^2.3.1",
    "grunt-webpack": "^3.1.3",
+    "grunt-zip": "^0.18.2",
    "html-webpack-plugin": "^3.2.0",
    "imports-loader": "^0.8.0",
    "ink-docstrap": "^1.3.2",
    "jsdoc-babel": "^0.5.0",
-    "mini-css-extract-plugin": "^0.5.0",
+    "mini-css-extract-plugin": "^0.6.0",
    "nightwatch": "^1.0.19",
    "node-sass": "^4.11.0",
    "postcss-css-variables": "^0.12.0",
@ -71,16 +72,15 @@
    "style-loader": "^0.23.1",
    "svg-url-loader": "^2.3.2",
    "url-loader": "^1.1.2",
-    "web-resource-inliner": "^4.3.1",
    "webpack": "^4.29.6",
-    "webpack-bundle-analyzer": "^3.1.0",
-    "webpack-dev-server": "^3.2.1",
+    "webpack-bundle-analyzer": "^3.3.2",
+    "webpack-dev-server": "^3.3.1",
    "webpack-node-externals": "^1.7.2",
    "worker-loader": "^2.0.0"
  },
  "dependencies": {
-    "@babel/polyfill": "^7.4.0",
-    "@babel/runtime": "^7.4.2",
+    "@babel/polyfill": "^7.4.3",
+    "@babel/runtime": "^7.4.3",
    "arrive": "^2.4.1",
    "babel-plugin-transform-builtin-extend": "1.1.2",
    "bcryptjs": "^2.4.3",
@ -92,7 +92,7 @@
    "bson": "^4.0.2",
    "chi-squared": "^1.1.0",
    "clippyjs": "0.0.3",
-    "core-js": "^3.0.0",
+    "core-js": "^3.0.1",
    "crypto-api": "^0.8.3",
    "crypto-js": "^3.1.9-1",
    "ctph.js": "0.0.5",
@ -107,8 +107,8 @@
    "file-saver": "^2.0.1",
    "geodesy": "^1.1.3",
    "highlight.js": "^9.15.6",
-    "jimp": "^0.6.0",
-    "jquery": "^3.3.1",
+    "jimp": "^0.6.1",
+    "jquery": "3.3.1",
    "js-crc": "^0.2.0",
    "js-sha3": "^0.8.0",
    "jsesc": "^2.5.2",
@ -130,7 +130,7 @@
    "notepack.io": "^2.2.0",
    "nwmatcher": "^1.4.4",
    "otp": "^0.1.3",
-    "popper.js": "^1.14.7",
+    "popper.js": "^1.15.0",
    "qr-image": "^3.2.0",
    "scryptsy": "^2.0.0",
    "snackbarjs": "^1.1.0",
--- a/src/core/ChefWorker.js
+++ b/src/core/ChefWorker.js
@ -178,7 +178,7 @@ self.loadRequiredModules = function(recipeConfig) {
        if (!OpModules.hasOwnProperty(module)) {
            log.info(`Loading ${module} module`);
            self.sendStatusMessage(`Loading ${module} module`);
-            self.importScripts(`${self.docURL}/${module}.js`);
+            self.importScripts(`${self.docURL}/modules/${module}.js`);
            self.sendStatusMessage("");
        }
    });
--- a/src/core/operations/RegularExpression.mjs
+++ b/src/core/operations/RegularExpression.mjs
@ -230,6 +230,7 @@ function regexHighlight (input, regex, displayTotal) {
        title = "",
        hl = 1,
        total = 0;
+    const captureGroups = [];

    output = input.replace(regex, (match, ...args) => {
        args.pop(); // Throw away full string
@ -247,9 +248,15 @@ function regexHighlight (input, regex, displayTotal) {
        // Switch highlight
        hl = hl === 1 ? 2 : 1;

-        total++;
+        // Store highlighted match and replace with a placeholder
+        captureGroups.push(`<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`);
+        return `[cc_capture_group_${total++}]`;
+    });

-        return `<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`;
+    // Safely escape all remaining text, then replace placeholders
+    output = Utils.escapeHtml(output);
+    output = output.replace(/\[cc_capture_group_(\d+)\]/g, (_, i) => {
+        return captureGroups[i];
    });

    if (displayTotal)
--- a/src/web/ControlsWaiter.mjs
+++ b/src/web/ControlsWaiter.mjs
@ -338,7 +338,7 @@ class ControlsWaiter {
        const saveLink = this.generateStateUrl(true, true, null, "https://gchq.github.io/CyberChef/");

        if (reportBugInfo) {
-            reportBugInfo.innerHTML = `* Version: ${PKG_VERSION + (typeof INLINE === "undefined" ? "" : "s")}
+            reportBugInfo.innerHTML = `* Version: ${PKG_VERSION}
 * Compile time: ${COMPILE_TIME}
 * User-Agent:
 ${navigator.userAgent}
--- a/src/web/html/index.html
+++ b/src/web/html/index.html
@ -131,13 +131,6 @@
            };
            window.addEventListener("error", loadingErrorHandler);
        </script>
-        <% if (htmlWebpackPlugin.options.inline) { %>
-            <meta name="robots" content="noindex" />
-        <% } else { %>
-            <script type="application/ld+json">
-                <% print(JSON.stringify(require("../static/structuredData.json"))); %>
-            </script>
-        <% } %>
    </head>
    <body>
        <!-- Preloader overlay -->
@ -153,11 +146,7 @@
        <div id="content-wrapper">
            <div id="banner" class="row">
                <div class="col" style="text-align: left; padding-left: 10px;">
-                    <% if (htmlWebpackPlugin.options.inline) { %>
-                        <span>Version <%= htmlWebpackPlugin.options.version %></span>
-                    <% } else { %>
-                        <a href="cyberchef.htm" download>Download CyberChef <i class="material-icons">file_download</i></a>
-                    <% } %>
+                    <a href="CyberChef_v<%= htmlWebpackPlugin.options.version %>.zip" download>Download CyberChef <i class="material-icons">file_download</i></a>
                </div>
                <div class="col-md-6" id="notice-wrapper">
                    <span id="notice">