update memgraph

memgraph/example/dnslookup

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+HOST=${1:?DOMAIN or IP is empty}
+
+docker exec -i memgraph_memgraph_1 mgconsole -output-format=csv <<- _CQL_ | sed -e 's/"//g' | tail -n +2 | tr '[],[]' ' ' | shuf | gawk -f /dev/fd/3 3<<- "_AWK_" | column -t -i1 -p2 -r3 -H1,2 | sed 's/─/& /'
+	MATCH p=(n)-[*]->(m)
+	WHERE any(n in nodes(p) where n.name = '$HOST') AND not exists(()-->(n)) AND not exists((m)-->())
+	UNWIND nodes(p) AS nn
+	WITH DISTINCT nn
+	CALL path.expand(nn,[">"],[],1,1) YIELD result
+	RETURN extract(i in nodes(result)|i.name);
+_CQL_
+	BEGIN {
+		split("", cache);
+		split("", roots);
+		idx=0;
+	}
+	{
+		if(!($1 in cache)) {
+			roots[$1] = cache[$1] = ++idx;
+		}
+		if(!($2 in cache)) {
+			cache[$2] = ++idx;
+		}
+		delete roots[$2];
+		print cache[$2], cache[$1], $2;
+	}
+	END {
+		print "0 -1 ."
+		for(root in roots) {
+			print cache[root], 0, root;
+		}
+	}
+_AWK_
+
+# vim: set noai noet:

memgraph/example/dump-elk.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+export PATH=/usr/local/bin:$PATH
+
+date=${1:-$(date -d yesterday +%F)}
+url=http://127.0.0.1:9200/
+index=logstash-${date//-/.}
+output=data/${date}.csv
+
+mkdir -p ${output%/*}
+
+elastic-query-export -c $url -i $index -o $output -q '+project:dns -_exists_:message +type:(A CNAME)' -fields '@timestamp,region,client,server,type,query,answer,ttl'
+
+gzip $output

memgraph/example/load-csv.cql

@@ -0,0 +1,27 @@
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH DISTINCT row.query AS query
+MERGE (d:Domain {name: query});
+
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH DISTINCT row.answer AS answer WHERE row.type = 'CNAME'
+MERGE (d:Domain {name: answer});
+
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH DISTINCT row.answer AS answer WHERE row.type = 'A'
+MERGE (i:IPv4 {name: answer});
+
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH row WHERE row.type = 'A'
+MATCH (d:Domain {name: row.query}), (i:IPv4 {name: row.answer})
+MERGE (d)-[r:A]->(i)
+ON CREATE SET r.created_at = timestamp()/1000000
+CALL date.parse(replace(row.`@timestamp`, "Z", ""), "s", "%Y-%m-%dT%H:%M:%S.%f", "UTC") YIELD parsed
+SET r.updated_at = parsed;
+
+LOAD CSV FROM "/path/to/dns.csv.gz" WITH HEADER AS row
+WITH row WHERE row.type = 'CNAME'
+MATCH (d1:Domain {name: row.query}), (d2:Domain {name: row.answer})
+MERGE (d1)-[r:CNAME]->(d2)
+ON CREATE SET r.created_at = timestamp()/1000000
+CALL date.parse(replace(row.`@timestamp`, "Z", ""), "s", "%Y-%m-%dT%H:%M:%S.%f", "UTC") YIELD parsed
+SET r.updated_at = parsed;

memgraph/example/load-csv.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+FILE=$(date -d yesterday +%F).csv.gz
+SRC_FILE=/data/dns/data/$FILE
+DST_FILE=/path/to/dns.csv.gz
+
+echo "$(date +%FT%T) GET $FILE"
+scp elk-us:$SRC_FILE $DST_FILE
+
+echo "$(date +%FT%T) LOAD CSV"
+docker exec -i memgraph_memgraph_1 mgconsole < load-csv.cql
+
+echo "$(date +%FT%T) DONE"