Adding missing lookup files

Author: nsreekanta

lookups/__mlspl_detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.mlmodel

@@ -0,0 +1,2 @@
+algo,model,options
+MLTKContainer,"{""__mlspl_type"": [""mltkc.MLTKContainer"", ""MLTKContainer""], ""dict"": {""endpoint_url"": ""https://localhost:62645"", ""out_params"": {""params"": {""mode"": ""stage"", ""algo"": ""detect_dns_data_exfiltration_using_pretrained_model_in_dsdl""}, ""args"": [""is_exfiltration"", ""src"", ""query"", ""rank""], ""target_variable"": [""is_exfiltration""], ""feature_variables"": [""src"", ""query"", ""rank""], ""model_name"": ""detect_dns_data_exfiltration_using_pretrained_model_in_dsdl"", ""algo_name"": ""MLTKContainer"", ""mlspl_limits"": {""handle_new_cat"": ""default"", ""max_distinct_cat_values"": ""100"", ""max_distinct_cat_values_for_classifiers"": ""100"", ""max_distinct_cat_values_for_scoring"": ""100"", ""max_fit_time"": ""600"", ""max_inputs"": ""100000"", ""max_memory_usage_mb"": ""4000"", ""max_model_size_mb"": ""30"", ""max_score_time"": ""600"", ""use_sampling"": ""true""}, ""kfold_cv"": null}, ""feature_variables"": [""src"", ""query"", ""rank""], ""target_variable"": ""is_exfiltration""}}","{""params"": {""mode"": ""stage"", ""algo"": ""detect_dns_data_exfiltration_using_pretrained_model_in_dsdl""}, ""args"": [""is_exfiltration"", ""src"", ""query"", ""rank""], ""target_variable"": [""is_exfiltration""], ""feature_variables"": [""src"", ""query"", ""rank""], ""model_name"": ""detect_dns_data_exfiltration_using_pretrained_model_in_dsdl"", ""algo_name"": ""MLTKContainer"", ""mlspl_limits"": {""handle_new_cat"": ""default"", ""max_distinct_cat_values"": ""100"", ""max_distinct_cat_values_for_classifiers"": ""100"", ""max_distinct_cat_values_for_scoring"": ""100"", ""max_fit_time"": ""600"", ""max_inputs"": ""100000"", ""max_memory_usage_mb"": ""4000"", ""max_model_size_mb"": ""30"", ""max_score_time"": ""600"", ""use_sampling"": ""true""}, ""kfold_cv"": null}"

lookups/__mlspl_detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.yml

@@ -0,0 +1,4 @@
+description: Detect DNS Data Exfiltration using pretrained Model in DSDL
+filename: __mlspl_detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.mlmodel
+name: __mlspl_detect_dns_data_exfiltration_using_pretrained_model_in_dsdl
+case_sensitive_match: 'false'

notebooks/detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.ipynb

@@ -189,9 +189,9 @@
     "class DNSExfiltration(nn.Module):\n",
     "    def __init__(self,input_size):\n",
     "        super().__init__()\n",
-    "        self.layer_1 = nn.Linear(input_size, 256) \n",
-    "        self.layer_2 = nn.Linear(256, 256)\n",
-    "        self.layer_out = nn.Linear(256, 1) \n",
+    "        self.layer_1 = nn.Linear(input_size, 128) \n",
+    "        self.layer_2 = nn.Linear(128, 128)\n",
+    "        self.layer_out = nn.Linear(128, 1) \n",
     "        \n",
     "        \n",
     "        self.relu = nn.ReLU()\n",
@@ -212,7 +212,7 @@
     "print (len(list(string.printable.strip())))\n",
     "    \n",
     "def init(df,param):\n",
-    "    model = DNSExfiltration(99)\n",
+    "    model = DNSExfiltration(98)\n",
     "    model.load_state_dict(torch.load(MODEL_DIRECTORY+'detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.pt',map_location=torch.device('cpu')))\n",
     "    model = model.to('cpu')\n",
     "    model.eval()\n",
@@ -382,14 +382,11 @@
     "    # length of domain\n",
     "    df['len'] = df['request_without_domain'].apply(len)\n",
     "    \n",
-    "    # number of subdomains\n",
-    "    df['subdomains_count'] = df['request_without_domain'].apply(lambda x: len(str(x).split('.')))\n",
-    "    \n",
     "    # entropy\n",
     "    df['entropy'] = df['request_without_domain'].apply(lambda x: entropy(x))\n",
     "    \n",
     "    # take most-recent request\n",
-    "    recent_df = df.loc[df['count'] == 1]\n",
+    "    recent_df = df.loc[df['rank'] == 1]\n",
     "\n",
     "    # calculate feature by aggregating events\n",
     "\n",
@@ -401,10 +398,10 @@
     "    \n",
     "# apply model on processed dataframe to predict exfiltration\n",
     "def apply(model,df,param):\n",
-    "    df.drop(['_time'], axis=1,inplace=True)\n",
+    "    df.drop(['_time'], axis=1,inplace=True, errors='ignore')\n",
     "    recent_df = prepare_input_df(df)\n",
-    "    input_df = recent_df.drop(['src' ,'query','count','request_without_domain','tld'], axis=1)\n",
-    "    recent_df.drop(['request_without_domain','tld','len','subdomains_count','entropy','size_avg','entropy_avg'], axis=1, inplace=True)\n",
+    "    input_df = recent_df.drop(['src' ,'query','rank','request_without_domain','tld'], axis=1)\n",
+    "    recent_df.drop(['request_without_domain','tld','len','entropy','size_avg','entropy_avg'], axis=1, inplace=True)\n",
     "    recent_df.drop(range(0, 94),axis=1,inplace=True)\n",
     "    input_tensor = torch.FloatTensor(input_df.values)\n",
     "    dataloader = DataLoader(input_tensor, shuffle=True, batch_size=256)\n",
@@ -416,7 +413,7 @@
     "    text_rows.clear()\n",
     "    size_avg.clear()\n",
     "    entropy_avg.clear()\n",
-    "    output = pd.merge(recent_df,df,on=['src','query','count'],how='right')\n",
+    "    output = pd.merge(recent_df,df,on=['src','query','rank'],how='right')\n",
     "    return output"
    ]
   },
@@ -461,7 +458,7 @@
    "source": [
     "# load model from name in expected convention \"<algo_name>_<model_name>\"\n",
     "def load(name):\n",
-    "    model = DNSExfiltration(99)\n",
+    "    model = DNSExfiltration(98)\n",
     "    model.load_state_dict(torch.load(MODEL_DIRECTORY+'detect_dns_data_exfiltration_using_pretrained_model_in_dsdl.pt',map_location=torch.device('cpu')))\n",
     "    model = model.to('cpu')\n",
     "    model.eval()\n",