Repository: FormalLanguageConstrainedPathQuerying/CFPQ_Data (language: python)

Issue:
Update



Discussion / hints:

Fix patch:
diff --git a/cfpq_data/data/MemoryAliases/Graphs/gzip.txt_meta.json b/cfpq_data/data/MemoryAliases/Graphs/gzip.txt_meta.json
index 4ab236cb..b204c057 100644
--- a/cfpq_data/data/MemoryAliases/Graphs/gzip.txt_meta.json
+++ b/cfpq_data/data/MemoryAliases/Graphs/gzip.txt_meta.json
@@ -1,6 +1,6 @@
 {
     "name": "gzip.txt.xml",
-    "path": "data/MemoryAliases/Graphs",
+    "path": "/home/viabzalov/GitReps/CFPQ_Data/cfpq_data/data/MemoryAliases/Graphs/gzip.txt.xml",
     "version": "0.0.0",
     "vertices": 2687,
     "edges": 2293,
diff --git a/cfpq_data/tools/__init__.py b/cfpq_data/tools/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/cfpq_data/tools/redis_rdf/README.md b/cfpq_data/tools/redis_rdf/README.md
deleted file mode 100644
index 1258b3e0..00000000
--- a/cfpq_data/tools/redis_rdf/README.md
+++ /dev/null
@@ -1,16 +0,0 @@
-### RDF loader in Redis Graph
-
-### Example of usage
-
-``` 
-$ python3 main.py file examples/graph.xml graph
-```
-``` 
-$ python3 main.py dir examples
-```
-
-### Help 
-
-``` 
-$ python3 main.py -h 
-```
diff --git a/cfpq_data/tools/redis_rdf/__init__.py b/cfpq_data/tools/redis_rdf/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/cfpq_data/tools/redis_rdf/examples/graph.xml b/cfpq_data/tools/redis_rdf/examples/graph.xml
deleted file mode 100644
index d414c36d..00000000
--- a/cfpq_data/tools/redis_rdf/examples/graph.xml
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<rdf:RDF
-  xmlns:yacc="http://yacc/"
-  xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
->
-  <rdf:Description rdf:about="http://yacc/A">
-    <yacc:parentOf>
-      <rdf:Description rdf:about="http://yacc/B">
-        <yacc:parentOf rdf:resource="http://yacc/C"/>
-      </rdf:Description>
-    </yacc:parentOf>
-  </rdf:Description>
-  <rdf:Description rdf:about="http://yacc/C">
-    <yacc:friendOf rdf:resource="http://yacc/B"/>
-    <yacc:parentOf rdf:resource="http://yacc/A"/>
-  </rdf:Description>
-</rdf:RDF>
diff --git a/cfpq_data/tools/redis_rdf/main.py b/cfpq_data/tools/redis_rdf/main.py
deleted file mode 100755
index f7444871..00000000
--- a/cfpq_data/tools/redis_rdf/main.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import os
-from argparse import ArgumentParser
-
-from src.redis_loader import load
-
-
-def main():
-    parser = ArgumentParser('Load rdf into RedisGraph')
-
-    parser.add_argument('--host', help='redis host name', default='localhost')
-    parser.add_argument('--port', help='redis port', default=6379)
-
-    subparsers = parser.add_subparsers()
-
-    parser_file = subparsers.add_parser('file')
-    parser_file.add_argument('RDF_PATH', help='rdf graph path')
-    parser_file.add_argument('GRAPH_NAME', help='redis graph name')
-
-    parser_dir = subparsers.add_parser('dir')
-    parser_dir.add_argument('dir_path')
-
-    args = parser.parse_args()
-    # logging.disable(logging.WARNING)
-
-    if 'dir_path' in args:
-        for file in os.listdir(args.dir_path):
-            load(f'{args.dir_path}/{file}', file, args.host, args.port)
-    else:
-        load(args.RDF_PATH, args.GRAPH_NAME, args.host, args.port)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/cfpq_data/tools/redis_rdf/requirements.txt b/cfpq_data/tools/redis_rdf/requirements.txt
deleted file mode 100644
index b2d73034..00000000
--- a/cfpq_data/tools/redis_rdf/requirements.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-pandas
-tqdm==4.36.1
-rdflib==4.2.2
-redis==3.3.10
-git+https://github.com/RedisGraph/redisgraph-py.git@275cf4c7609a061e55b538e10efbc02c91c090e7
\ No newline at end of file
diff --git a/cfpq_data/tools/redis_rdf/src/__init__.py b/cfpq_data/tools/redis_rdf/src/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/cfpq_data/tools/redis_rdf/src/redis_loader/__init__.py b/cfpq_data/tools/redis_rdf/src/redis_loader/__init__.py
deleted file mode 100644
index 5d173dba..00000000
--- a/cfpq_data/tools/redis_rdf/src/redis_loader/__init__.py
+++ /dev/null
@@ -1,1 +0,0 @@
-from .loader import load
\ No newline at end of file
diff --git a/cfpq_data/tools/redis_rdf/src/redis_loader/graph.py b/cfpq_data/tools/redis_rdf/src/redis_loader/graph.py
deleted file mode 100644
index 4f9f028e..00000000
--- a/cfpq_data/tools/redis_rdf/src/redis_loader/graph.py
+++ /dev/null
@@ -1,39 +0,0 @@
-import redisgraph
-
-
-class Graph(redisgraph.Graph):
-    def __init__(self, name, redis_con):
-        super().__init__(name, redis_con)
-
-    def commit_edges(self):
-        """
-        Create edges with existing nodes.
-        """
-        if len(self.nodes) == 0 and len(self.edges) == 0:
-            return None
-
-        query = 'MATCH '
-        for _, node in self.nodes.items():
-            query += str(node) + ','
-
-        # Discard leading comma.
-        if query[-1] is ',':
-            query = query[:-1]
-
-        query += ' CREATE '
-
-        query += ','.join([str(edge) for edge in self.edges])
-
-        # Discard leading comma.
-        if query[-1] is ',':
-            query = query[:-1]
-
-        return self.query(query)
-
-    def flush_edges(self):
-        """
-        Commit the edges and reset the edges and nodes to zero length
-        """
-        self.commit_edges()
-        self.nodes = {}
-        self.edges = []
diff --git a/cfpq_data/tools/redis_rdf/src/redis_loader/loader.py b/cfpq_data/tools/redis_rdf/src/redis_loader/loader.py
deleted file mode 100644
index fb3da145..00000000
--- a/cfpq_data/tools/redis_rdf/src/redis_loader/loader.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import hashlib
-
-from redis import Redis
-from redisgraph import Node as RedisNode, Edge as RedisEdge
-from tqdm import tqdm
-
-from .graph import Graph as RedisGraph
-from .triplet_loader import load_rdf_graph
-
-BLOCK_SIZE = 100
-
-def make_node(value: str, alias=None):
-    return RedisNode(
-        label='Node', alias=alias, properties={
-            'value': value,
-            'sha256': hashlib.sha256(str(value).encode('utf-8')).hexdigest()
-        })
-
-def load_in_redis(rdf_graph, redis_graph: RedisGraph):
-    all_nodes = dict()
-
-    # Add all nodes from file to hash map
-    for subj, _, obj in rdf_graph:
-        all_nodes.setdefault(str(subj), make_node(str(subj)))
-        all_nodes.setdefault(str(obj), make_node(str(obj)))
-
-    print(f'{redis_graph.name}: Add all nodes to redis graph')
-    for k, v in tqdm(all_nodes.items()):
-        redis_graph.add_node(v)
-        if len(redis_graph.nodes) > BLOCK_SIZE:
-            redis_graph.flush()
-    redis_graph.flush()
-
-    # Create index over Nodes
-    redis_graph.query("CREATE INDEX ON :Node(sha256)")
-
-    print(f'{redis_graph.name}: Add edges to existing nodes')
-    for subj, pred, obj in tqdm(rdf_graph):
-        edge = RedisEdge(all_nodes[str(subj)], pred, all_nodes[str(obj)])
-        edge_r = RedisEdge(all_nodes[str(obj)], f'{pred}_r', all_nodes[str(subj)])
-        redis_graph.add_node(all_nodes[str(subj)])
-        redis_graph.add_node(all_nodes[str(obj)])
-        redis_graph.add_edge(edge)
-        redis_graph.add_edge(edge_r)
-        if len(redis_graph.edges) > BLOCK_SIZE:
-            redis_graph.flush_edges()
-    redis_graph.flush_edges()
-
-
-def load(rdf_file: str, redis_graph_name: str, redis_host: str, redis_port: int):
-    rdf_graph = load_rdf_graph(rdf_file)
-
-    redis_connector = Redis(host=redis_host, port=redis_port)
-    redis_graph = RedisGraph(redis_graph_name, redis_connector)
-
-    load_in_redis(rdf_graph, redis_graph)
diff --git a/cfpq_data/tools/redis_rdf/src/redis_loader/triplet_loader.py b/cfpq_data/tools/redis_rdf/src/redis_loader/triplet_loader.py
deleted file mode 100644
index 6add6e6f..00000000
--- a/cfpq_data/tools/redis_rdf/src/redis_loader/triplet_loader.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import pathlib
-
-import rdflib
-
-
-def load_triplets(filename: str):
-    triplets = list()
-    with open(filename, mode='r') as file:
-        for line in file:
-            try:
-                subj, pred, obj = line[:-1].split()
-                triplets.append((subj, pred, obj))
-            except ValueError:
-                continue
-    return triplets
-
-
-def uri_to_name(identifier, rdf_graph: rdflib.Graph):
-    if isinstance(identifier, rdflib.URIRef):
-        prefix, namespace, name = rdf_graph.compute_qname(identifier)
-        return name
-    return identifier
-
-
-def load_rdf_graph(rdf_file: str):
-    rdf_graph = rdflib.Graph()
-    rdf_graph.load(rdf_file)
-    return list(map(lambda spo: (spo[0], uri_to_name(spo[1], rdf_graph), spo[2]), iter(rdf_graph)))
diff --git a/cfpq_data/tools/redis_rdf/src/utils/__init__.py b/cfpq_data/tools/redis_rdf/src/utils/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/docs/README.md b/docs/README.md
index 45c27ae0..e524531a 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -16,23 +16,7 @@ python3
 >>> help(cfpq_data)
 ```
 
-## Integration with graph DBs
-
-We provide a set of scripts to simplify data loading into some popular graph databases.
-
-### RedisGraph
-
-**Note! At the moment redis_rdf is not part of the cfpq_data project and can only be used after downloading the
-repository directly**
-
-The dataset can be loaded to RedisGraph with ```cfpq_data/tools/redis_rdf```, for example:
-
-```
-python3 cfpq_data/tools/redis_rdf/main.py --host [HOST] --port [PORT] dir <path to folder with graphs>
-python3 cfpq_data/tools/redis_rdf/main.py --host [HOST] --port [PORT] file <path to graph> <graph name>
-```
-
-### Neo4j
+### Integration with Neo4j
 
 To load RDF data into Neo4j database one can use [Neosemantix plugin for Neo4j](https://neo4j.com/labs/nsmtx-rdf/).
 


Test patch:
diff --git a/cfpq_data/tools/redis_rdf/tests/test_load.py b/cfpq_data/tools/redis_rdf/tests/test_load.py
deleted file mode 100644
index 44aaa5ac..00000000
--- a/cfpq_data/tools/redis_rdf/tests/test_load.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import logging
-import random
-import string
-from unittest import TestCase
-
-from redis import Redis
-from redisgraph import Edge as RedisEdge
-
-from src.redis_loader.graph import Graph as RedisGraph
-from src.redis_loader.loader import load_in_redis, make_node
-from src.redis_loader.triplet_loader import load_rdf_graph
-
-
-def print_edge_with_alias(edge: RedisEdge, alias: str):
-    return f'{edge.src_node}-[{alias}: {edge.relation} {edge.toString()}]->{edge.dest_node}'
-
-
-class TestLoad(TestCase):
-    def setUp(self):
-        logging.disable(logging.WARNING)
-        self.redis_connector = Redis(host='localhost', port=6380)
-
-    def testLoadTxt(self):
-        self.loadAndCheck(load_rdf_graph('examples/graph.txt'))
-
-    def testLoadXml(self):
-        self.loadAndCheck(load_rdf_graph('examples/graph.xml'))
-
-    def loadAndCheck(self, rdf_graph):
-        redis_graph = RedisGraph(self.randomGraphName(), self.redis_connector)
-
-        # load
-        load_in_redis(rdf_graph, redis_graph)
-
-        # check every edge
-        for subj, pred, obj in rdf_graph:
-            self.assertTrue(self.checkEdgeExist(redis_graph, self.makeEdge(subj, pred, obj)))
-            self.assertTrue(self.checkEdgeExist(redis_graph, self.makeEdge(obj, f'{pred}_r', subj)))
-
-        # check total edges count
-        self.assertEqual(len(rdf_graph) * 2, self.countEdges(redis_graph))
-
-    @staticmethod
-    def makeEdge(subj, pred, obj):
-        return RedisEdge(make_node(str(subj), 'src'), pred, make_node(str(obj)), 'dst')
-
-    @staticmethod
-    def checkEdgeExist(redis_graph: RedisGraph, edge: RedisEdge):
-        query = f'MATCH {print_edge_with_alias(edge, "edge")} RETURN COUNT(edge)'
-        count = redis_graph.query(query).result_set[0][0]
-        return count > 0
-
-    @staticmethod
-    def countEdges(redis_graph: RedisGraph):
-        query = f'MATCH ()-[r]->() RETURN COUNT(r)'
-        return redis_graph.query(query).result_set[0][0]
-
-    @staticmethod
-    def randomGraphName():
-        letters = string.ascii_lowercase
-        return ''.join(random.choice(letters) for i in range(10))