h4cker/dfir/threat-hunting-analysis-main/HTTP Nmap Indicator Detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from datetime import datetime, timedelta\n",
    "import matplotlib.pylab as plot\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read data from http Zeek (formerly known as Bro) logs\n",
    "with open(\"http.log\",'r') as infile:\n",
    "    file_data = infile.read()\n",
    "    \n",
    "# Split file by newlines\n",
    "file_data = file_data.split('\\n')\n",
    "\n",
    "# Remove comment lines\n",
    "http_data = []\n",
    "for line in file_data:\n",
    "    if line[0] is not None and line[0] != \"#\":\n",
    "        http_data.append(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"/ftv2lastnode.gif\": 2, \n",
      "  \"/ftv2mnode.gif\": 2, \n",
      "  \"/pics/play_button_27x27px.gif\": 4, \n",
      "  \"/led.asp\": 2, \n",
      "  \"/pics/gray_corner_rt_5x50px.gif\": 4, \n",
      "  \"/img/device.gif\": 4, \n",
      "  \"/ RTSP/1.\": 5, \n",
      "  \"/pics/gray_corner_lt_5x50px.gif\": 4, \n",
      "  \"/webserverconfig.asp\": 1, \n",
      "  \"/auth/logo2_516.gif\": 5, \n",
      "  \"/index.htm\": 1, \n",
      "  \"/syslogserverconfig.asp\": 2, \n",
      "  \"/logo2_516.gif\": 1, \n",
      "  \"/neighbor_cache_table.asp\": 2, \n",
      "  \"/generalinst.htm\": 1, \n",
      "  \"/view/temp.shtml\": 2, \n",
      "  \"/img/checkbox_nchk.gif\": 1, \n",
      "  \"/jscript/sysstatus.js\": 1, \n",
      "  \"/SetModSerial.html\": 1, \n",
      "  \"/logo3.gif\": 2, \n",
      "  \"/status.jsp\": 1, \n",
      "  \"/port_setting.asp\": 1, \n",
      "  \"/syslog_message.asp\": 1, \n",
      "  \"/logo2_EDS-508A.gif\": 1, \n",
      "  \"/port_setting_show.asp\": 1, \n",
      "  \"/jscript/statistics.js\": 3, \n",
      "  \"/images/off.gif\": 3, \n",
      "  \"/pics/line_corner_rb_5x5px.gif\": 4, \n",
      "  \"/sysstatus.asp\": 1, \n",
      "  \"/overview.asp\": 4, \n",
      "  \"/jscript/powerconfig.js\": 1, \n",
      "  \"/jscript/login.js\": 4, \n",
      "  \"/mac_address_table_setting.asp\": 4, \n",
      "  \"/.git/HEAD\": 11, \n",
      "  \"/setid.html\": 1, \n",
      "  \"/network_setting_ipv6.asp\": 1, \n",
      "  \"/activate_button.gif\": 10, \n",
      "  \"/goform/svLogin\": 3, \n",
      "  \"/ftv2plastnode.gif\": 1, \n",
      "  \"/ftv2folderopen.gif\": 2, \n",
      "  \"/tasktracker.jsp\": 1, \n",
      "  \"/spconfig.asp\": 4, \n",
      "  \"/pics/line_corner_lt_5x5px.gif\": 4, \n",
      "  \"/pdmonitor.htm\": 1, \n",
      "  \"/settable.html\": 1, \n",
      "  \"/spconnect.asp\": 2, \n",
      "  \"/setdesc.html\": 1, \n",
      "  \"/jscript/ipconfig.js\": 3, \n",
      "  \"/syslogging.asp\": 1, \n",
      "  \"/images/connect.gif\": 2, \n",
      "  \"/jobtracker.jsp\": 1, \n",
      "  \"/ftv2pnode.gif\": 1, \n",
      "  \"/eip_setting.asp\": 1, \n",
      "  \"/ftv2mlastnode.gif\": 2, \n",
      "  \"/garp_timer_setting.asp\": 1, \n",
      "  \"/auth/md5.js\": 13, \n",
      "  \"/incl/activeX.js\": 4, \n",
      "  \"/pics/line_corner_lb_5x5px.gif\": 4, \n",
      "  \"/css/win_ns.css\": 6, \n",
      "  \"/browseDirectory.jsp\": 1, \n",
      "  \"/jscript/spconnect.js\": 2, \n",
      "  \"/modbus_setting.asp\": 1, \n",
      "  \"/master.jsp\": 1, \n",
      "  \"/hwinstall.htm\": 1, \n",
      "  \"/md5.js\": 3, \n",
      "  \"/snmpconfig.asp\": 3, \n",
      "  \"/bg.gif\": 2, \n",
      "  \"/url/ups1.scc\": 1, \n",
      "  \"/\": 187, \n",
      "  \"/rs-status\": 1, \n",
      "  \"/home.asp\": 10, \n",
      "  \"/bus_configuration.htm\": 1, \n",
      "  \"/pics/line_t_100x5px.gif\": 4, \n",
      "  \"/jscript/nfsserverconfig.js\": 1, \n",
      "  \"/setip.html\": 1, \n",
      "  \"/img/pxclogo.gif\": 20, \n",
      "  \"/robots.txt\": 11, \n",
      "  \"/port_setting726.asp\": 2, \n",
      "  \"/name.asp\": 2, \n",
      "  \"/dip_switch_setting.asp\": 1, \n",
      "  \"/jscript/powerunitmanage.js\": 1, \n",
      "  \"/jscript/syslogserverconfig.js\": 2, \n",
      "  \"/local_diagnostics.htm\": 1, \n",
      "  \"/jscript/slidemenu.js\": 6, \n",
      "  \"/powermanage.asp\": 1, \n",
      "  \"/ipconfig.asp\": 3, \n",
      "  \"/jscript/util.js\": 4, \n",
      "  \"/deviceinfo.htm\": 2, \n",
      "  \"/auth/led_auth.asp\": 13, \n",
      "  \"/images/ws_button3.gif\": 4, \n",
      "  \"/flumemaster.jsp\": 1, \n",
      "  \"/goform/EventLogList\": 2, \n",
      "  \"/settimeouts.html\": 1, \n",
      "  \"/tagbase_vlan_setting_show.asp\": 1, \n",
      "  \"12.1.2\": 2, \n",
      "  \"/img/device_s.gif\": 20, \n",
      "  \"/ftv2folderclosed.gif\": 2, \n",
      "  \"/favicon.ico\": 81, \n",
      "  \"/showstatus.html\": 1, \n",
      "  \"/techdata.htm\": 2, \n",
      "  \"/pics/blank.gif\": 4, \n",
      "  \"/dfshealth.jsp\": 1, \n",
      "  \"/images/block.gif\": 3, \n",
      "  \"/css/common.css\": 6, \n",
      "  \"/ftv2vertline.gif\": 2, \n",
      "  \"/stserial.asp\": 80, \n",
      "  \"/nice ports,/Trinity.txt.bak\": 8, \n",
      "  \"/port_setting_show726.asp\": 2, \n",
      "  \"/userloggedonlist.asp\": 1, \n",
      "  \"/reset_button.gif\": 2, \n",
      "  \"/login.asp\": 5, \n",
      "  \"/monitor_statistic_cnt_show.asp\": 2, \n",
      "  \"/getstatus.html\": 4737, \n",
      "  \"/ups1.scc\": 1, \n",
      "  \"/auth/topplan_auth.asp\": 15, \n",
      "  \"/pics/logo_70x29px.gif\": 4, \n",
      "  \"/view\": 1, \n",
      "  \"/ws_button3.gif\": 2, \n",
      "  \"sip:nm SIP/2.\": 4, \n",
      "  \"/pics/space.gif\": 4, \n",
      "  \"/jscript/rhostaccessctrl.js\": 2, \n",
      "  \"/powerconfig.asp\": 1, \n",
      "  \"/tagbase_vlan_setting.asp\": 1, \n",
      "  \"/ftv2node.gif\": 2, \n",
      "  \"/remote_diagnostics.htm\": 1, \n",
      "  \"/images/on.gif\": 2, \n",
      "  \"/jscript/webserverconfig.js\": 1, \n",
      "  \"/auth/loginin.gif\": 13, \n",
      "  \"/left_down_logo.asp\": 2, \n",
      "  \"/auth/accountpassword.asp\": 13, \n",
      "  \"/ftv2blank.gif\": 2, \n",
      "  \"/logo1.gif\": 2, \n",
      "  \"/images/logo.gif\": 4, \n",
      "  \"/rhostaccessctrl.asp\": 2, \n",
      "  \"/ipconfig.htm\": 2, \n",
      "  \"/auth/logo1.gif\": 13, \n",
      "  \"/view/index.shtml\": 7, \n",
      "  \"/ddnsconfig.asp\": 2, \n",
      "  \"/tcpserviceconfig.asp\": 1, \n",
      "  \"/auth/logo2_EDS-508A.gif\": 8, \n",
      "  \"/auth/name_auth.asp\": 13, \n",
      "  \"/monitor_port.asp\": 2, \n",
      "  \"/css/digistyle.css\": 4, \n",
      "  \"/pics/stop_button_27x27px.gif\": 4, \n",
      "  \"/pcp_configuration.htm\": 1, \n",
      "  \"/pics/line_b_100x5px.gif\": 4, \n",
      "  \"-\": 45, \n",
      "  \"/img/checkbox_chk.gif\": 1, \n",
      "  \"/view/view.shtml\": 4, \n",
      "  \"/img/hw_installation.gif\": 1, \n",
      "  \"/jscript/spconfig.js\": 3, \n",
      "  \"/jscript/snmpconfig.js\": 3, \n",
      "  \"/view/\": 9, \n",
      "  \"/vlan_set.asp\": 1, \n",
      "  \"/mjpg/video.mjpg\": 7, \n",
      "  \"/log_setting.asp\": 2, \n",
      "  \"/smtpconfig.asp\": 1, \n",
      "  \"/jscript/validation.js\": 4, \n",
      "  \"/clear_button.gif\": 2, \n",
      "  \"/phoenix_fl.js\": 20, \n",
      "  \"/jscript/smtpconfig.js\": 1, \n",
      "  \"/services.htm\": 3, \n",
      "  \"/pics/line_corner_rt_5x5px.gif\": 4, \n",
      "  \"/phoenix_fl.css\": 20, \n",
      "  \"/nfsserverconfig.asp\": 1, \n",
      "  \"/jscript/syslogging.js\": 1, \n",
      "  \"/auth/logo3.gif\": 13, \n",
      "  \"/stnetwork.asp\": 1, \n",
      "  \"/pics/gray_t_5x50px.gif\": 4, \n",
      "  \"/auth/auth.asp\": 23, \n",
      "  \"/jscript/default.js\": 4, \n",
      "  \"/d4-43.js\": 2, \n",
      "  \"/left.asp\": 2, \n",
      "  \"/jscript/ddnsconfig.js\": 2, \n",
      "  \"/img/sel.gif\": 16, \n",
      "  \"/ethernetconfig.asp\": 1\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Let's stack uris\n",
    "uris = {}\n",
    "for line in http_data:\n",
    "    if len(line.split('\\t')) > 9:\n",
    "        uri = line.split('\\t')[9].split('?')[0].split('&')[0]\n",
    "        if uri not in uris.keys():\n",
    "            uris[uri] = 1\n",
    "        else:\n",
    "            uris[uri] += 1\n",
    "\n",
    "print(json.dumps(uris,indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:39.0) Gecko/20100101 Firefox/39.0\": 327, \n",
      "  \"Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)\": 171, \n",
      "  \"-\": 103, \n",
      "  \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2490.64 Safari/537.36\": 5045, \n",
      "  \"Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:28.0) Gecko/20100101 Firefox/28.0\": 12, \n",
      "  \"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0\": 99, \n",
      "  \"Wget/1.16.1 (linux-gnu)\": 1\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Let's stack user agents\n",
    "user_agents = {}\n",
    "for line in http_data:\n",
    "    if len(line.split('\\t')) > 12:\n",
    "        user_agent = line.split('\\t')[11]\n",
    "        if user_agent not in user_agents.keys():\n",
    "            user_agents[user_agent] = 1\n",
    "        else:\n",
    "            user_agents[user_agent] += 1\n",
    "\n",
    "print(json.dumps(user_agents,indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"192.168.2.42\": {\n",
      "    \"192.168.88.115\": {\n",
      "      \"1445425464.684730\": 1, \n",
      "      \"1445425489.066291\": 1, \n",
      "      \"1445425456.492019\": 1, \n",
      "      \"1445425472.897110\": 1, \n",
      "      \"1445425505.330748\": 1, \n",
      "      \"1445425497.221008\": 1, \n",
      "      \"1445425472.798104\": 1, \n",
      "      \"1445425464.734434\": 1, \n",
      "      \"1445425489.264708\": 1, \n",
      "      \"1445425481.058994\": 1, \n",
      "      \"1445425456.491738\": 1, \n",
      "      \"1445425456.492152\": 1, \n",
      "      \"1445425464.684854\": 1, \n",
      "      \"1445425521.550031\": 1, \n",
      "      \"1445425456.491596\": 1, \n",
      "      \"1445425456.492557\": 1, \n",
      "      \"1445425513.438493\": 1, \n",
      "      \"1445425480.908743\": 1\n",
      "    }\n",
      "  }, \n",
      "  \"192.168.2.64\": {\n",
      "    \"192.168.88.25\": {\n",
      "      \"1445422296.875484\": 1, \n",
      "      \"1445422290.967679\": 1, \n",
      "      \"1445422289.381463\": 1, \n",
      "      \"1445422289.591706\": 1, \n",
      "      \"1445422290.459930\": 1, \n",
      "      \"1445422323.002866\": 1, \n",
      "      \"1445422289.808332\": 1, \n",
      "      \"1445422291.185004\": 1, \n",
      "      \"1445422290.239258\": 1, \n",
      "      \"1445422296.668006\": 1, \n",
      "      \"1445422290.239120\": 1, \n",
      "      \"1445422292.854650\": 1, \n",
      "      \"1445422290.678547\": 1, \n",
      "      \"1445422290.020238\": 1, \n",
      "      \"1445422314.053171\": 1, \n",
      "      \"1445422313.799369\": 1, \n",
      "      \"1445422291.184861\": 1, \n",
      "      \"1445422300.715145\": 1\n",
      "    }, \n",
      "    \"192.168.88.115\": {\n",
      "      \"1445422321.290313\": 1, \n",
      "      \"1445422300.766784\": 1, \n",
      "      \"1445422320.650723\": 1, \n",
      "      \"1445422321.503861\": 1, \n",
      "      \"1445422300.184951\": 1, \n",
      "      \"1445422321.928420\": 1, \n",
      "      \"1445422320.867814\": 1, \n",
      "      \"1445422291.938518\": 1, \n",
      "      \"1445422322.355297\": 1, \n",
      "      \"1445422292.693354\": 1, \n",
      "      \"1445422321.713691\": 1, \n",
      "      \"1445422316.046787\": 1, \n",
      "      \"1445422322.142027\": 1, \n",
      "      \"1445422321.077807\": 1, \n",
      "      \"1445422291.454377\": 1\n",
      "    }, \n",
      "    \"192.168.88.20\": {\n",
      "      \"1445422298.992223\": 1, \n",
      "      \"1445422291.885333\": 1, \n",
      "      \"1445422302.855427\": 1, \n",
      "      \"1445422300.497165\": 1, \n",
      "      \"1445422299.414991\": 1, \n",
      "      \"1445422315.698055\": 1, \n",
      "      \"1445422300.287326\": 1, \n",
      "      \"1445422290.968135\": 1, \n",
      "      \"1445422299.207919\": 1, \n",
      "      \"1445422299.839276\": 1, \n",
      "      \"1445422298.777344\": 1, \n",
      "      \"1445422300.078390\": 1, \n",
      "      \"1445422313.532961\": 1, \n",
      "      \"1445422299.628075\": 1\n",
      "    }, \n",
      "    \"192.168.88.100\": {\n",
      "      \"1445422308.102295\": 1, \n",
      "      \"1445422289.380025\": 1, \n",
      "      \"1445422290.915620\": 1, \n",
      "      \"1445422297.138751\": 1, \n",
      "      \"1445422290.513640\": 1\n",
      "    }, \n",
      "    \"192.168.88.51\": {\n",
      "      \"1445422295.870961\": 1, \n",
      "      \"1445422300.023159\": 1, \n",
      "      \"1445422320.920019\": 1, \n",
      "      \"1445422303.707740\": 1, \n",
      "      \"1445422296.667868\": 1, \n",
      "      \"1445422289.754808\": 1, \n",
      "      \"1445422299.364282\": 1, \n",
      "      \"1445422297.667609\": 1, \n",
      "      \"1445422292.639583\": 1, \n",
      "      \"1445422298.789861\": 1, \n",
      "      \"1445422289.381938\": 1, \n",
      "      \"1445422290.520664\": 1, \n",
      "      \"1445422296.027733\": 1, \n",
      "      \"1445422300.212852\": 1, \n",
      "      \"1445422292.587508\": 1, \n",
      "      \"1445422300.341810\": 1, \n",
      "      \"1445422295.554722\": 1, \n",
      "      \"1445422299.694729\": 1, \n",
      "      \"1445422295.714594\": 1, \n",
      "      \"1445422300.498336\": 1, \n",
      "      \"1445422293.066879\": 1, \n",
      "      \"1445422292.476080\": 1, \n",
      "      \"1445422299.696478\": 1, \n",
      "      \"1445422289.592098\": 1, \n",
      "      \"1445422303.873797\": 1, \n",
      "      \"1445422300.660455\": 1, \n",
      "      \"1445422290.349694\": 1, \n",
      "      \"1445422299.260279\": 1, \n",
      "      \"1445422299.840329\": 1, \n",
      "      \"1445422289.385586\": 1, \n",
      "      \"1445422296.188602\": 1, \n",
      "      \"1445422299.518622\": 1, \n",
      "      \"1445422298.727806\": 1, \n",
      "      \"1445422320.466621\": 1, \n",
      "      \"1445422296.506938\": 1, \n",
      "      \"1445422296.349914\": 1, \n",
      "      \"1445422323.263679\": 1, \n",
      "      \"1445422296.824060\": 1, \n",
      "      \"1445422303.927905\": 1\n",
      "    }, \n",
      "    \"192.168.88.49\": {\n",
      "      \"1445422302.534936\": 1, \n",
      "      \"1445422292.047762\": 1, \n",
      "      \"1445422289.380561\": 1, \n",
      "      \"1445422302.965697\": 1, \n",
      "      \"1445422302.746772\": 1, \n",
      "      \"1445422291.619375\": 1, \n",
      "      \"1445422303.183484\": 1, \n",
      "      \"1445422307.565998\": 1, \n",
      "      \"1445422301.635377\": 1, \n",
      "      \"1445422313.849169\": 1, \n",
      "      \"1445422302.111056\": 1, \n",
      "      \"1445422303.397388\": 1, \n",
      "      \"1445422302.325429\": 1, \n",
      "      \"1445422301.899644\": 1\n",
      "    }, \n",
      "    \"192.168.88.60\": {\n",
      "      \"1445422289.865632\": 1, \n",
      "      \"1445422289.591967\": 1, \n",
      "      \"1445422291.235170\": 1, \n",
      "      \"1445422291.885204\": 1, \n",
      "      \"1445422289.381938\": 1, \n",
      "      \"1445422291.018922\": 1, \n",
      "      \"1445422306.307627\": 1, \n",
      "      \"1445422290.565864\": 1, \n",
      "      \"1445422292.319808\": 1, \n",
      "      \"1445422299.890418\": 1, \n",
      "      \"1445422292.100843\": 1, \n",
      "      \"1445422289.381132\": 1, \n",
      "      \"1445422298.992366\": 1, \n",
      "      \"1445422291.454248\": 1, \n",
      "      \"1445422289.379891\": 1, \n",
      "      \"1445422289.865921\": 1, \n",
      "      \"1445422298.777468\": 1\n",
      "    }, \n",
      "    \"192.168.88.61\": {\n",
      "      \"1445422300.131605\": 1, \n",
      "      \"1445422289.591833\": 1, \n",
      "      \"1445422300.988103\": 1, \n",
      "      \"1445422292.798306\": 1, \n",
      "      \"1445422289.866199\": 1, \n",
      "      \"1445422290.915767\": 1, \n",
      "      \"1445422299.679622\": 1, \n",
      "      \"1445422297.244478\": 1, \n",
      "      \"1445422300.766659\": 1, \n",
      "      \"1445422301.201058\": 1, \n",
      "      \"1445422299.466633\": 1, \n",
      "      \"1445422293.119720\": 1, \n",
      "      \"1445422300.548608\": 1, \n",
      "      \"1445422299.890145\": 1, \n",
      "      \"1445422300.339324\": 1\n",
      "    }, \n",
      "    \"192.168.88.95\": {\n",
      "      \"1445422289.380290\": 1, \n",
      "      \"1445422344.783066\": 1, \n",
      "      \"1445422352.905377\": 1, \n",
      "      \"1445422317.744378\": 1, \n",
      "      \"1445422321.022581\": 1, \n",
      "      \"1445422320.387407\": 1, \n",
      "      \"1445422295.370559\": 1, \n",
      "      \"1445422309.529967\": 1, \n",
      "      \"1445422336.568033\": 1, \n",
      "      \"1445422320.490386\": 1, \n",
      "      \"1445422301.580724\": 1, \n",
      "      \"1445422337.822249\": 1, \n",
      "      \"1445422305.513430\": 1, \n",
      "      \"1445422348.751232\": 1, \n",
      "      \"1445422290.347162\": 1, \n",
      "      \"1445422289.380169\": 1\n",
      "    }\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "# Let's search for the nmap user agent\n",
    "suspicious_user_agents = ['Mozilla/5.0 (compatible; Nmap Scripting Engine; http://nmap.org/book/nse.html)']\n",
    "nmap_scanned_hosts = {}\n",
    "for line in http_data:\n",
    "    if len(line.split('\\t')) > 12:\n",
    "        timestamp = line.split('\\t')[0]\n",
    "        client = line.split('\\t')[2]\n",
    "        server = line.split('\\t')[4]\n",
    "        user_agent = line.split('\\t')[11]\n",
    "        if user_agent in suspicious_user_agents:\n",
    "            if client not in nmap_scanned_hosts.keys():\n",
    "                nmap_scanned_hosts[client] = {server:{timestamp:1}}\n",
    "            elif server not in nmap_scanned_hosts[client].keys():\n",
    "                nmap_scanned_hosts[client][server] = {timestamp: 1}\n",
    "            elif timestamp not in nmap_scanned_hosts[client][server].keys():\n",
    "                nmap_scanned_hosts[client][server][timestamp] = 1\n",
    "            else:\n",
    "                nmap_scanned_hosts[client][server][timestamp] += 1\n",
    "\n",
    "print(json.dumps(nmap_scanned_hosts,indent=2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "client ip,server ip,num requests\n",
      "192.168.2.42,192.168.88.115,18\n",
      "192.168.2.64,192.168.88.100,5\n",
      "192.168.2.64,192.168.88.115,15\n",
      "192.168.2.64,192.168.88.20,14\n",
      "192.168.2.64,192.168.88.25,18\n",
      "192.168.2.64,192.168.88.49,14\n",
      "192.168.2.64,192.168.88.51,39\n",
      "192.168.2.64,192.168.88.60,17\n",
      "192.168.2.64,192.168.88.61,15\n",
      "192.168.2.64,192.168.88.95,16\n"
     ]
    }
   ],
   "source": [
    "# Add up the number of requests the client made to the server\n",
    "print(\"client ip,server ip,num requests\")\n",
    "suspicious_hosts = {}\n",
    "for client in sorted(nmap_scanned_hosts.keys()):\n",
    "    for server in sorted(nmap_scanned_hosts[client].keys()):\n",
    "        print(client + \",\" + server + \",\" + str(len(nmap_scanned_hosts[client][server])))\n",
    "        if client not in suspicious_hosts.keys():\n",
    "            suspicious_hosts[client] = [server]\n",
    "        else:\n",
    "            suspicious_hosts[client].append(server)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Write CSV file out for display/distribution in excel\n",
    "with open('suspicious_http_records.csv','w') as outfile:\n",
    "    outfile.write(\"ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,trans_depth,method,host,uri,referrer,user_agent,request_body_len,response_body_len,status_code,status_msg,info_code,info_msg,filename,tags,username,password,proxied,orig_fuids,orig_mime_types,resp_fuids,resp_mime_types\\n\")\n",
    "    for line in http_data:\n",
    "        if len(line.split('\\t')) > 12:\n",
    "            timestamp = line.split('\\t')[0]\n",
    "            client = line.split('\\t')[2]\n",
    "            server = line.split('\\t')[4]\n",
    "            user_agent = line.split('\\t')[11]\n",
    "            uri = line.split('\\t')[9]\n",
    "            if client in suspicious_hosts.keys():\n",
    "                if server in suspicious_hosts[client]:\n",
    "                    outfile.write(\"\\\"\" + line.replace(\"\\t\",\"\\\",\\\"\") + \"\\\"\\n\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}