GraphScope阿里巴巴一站式大规模图计算系统

pip3 install graphscope

pip3 install -U pip

import os
import graphscope
# Setting an env for mounting test data from local disk,
# hence we can access data inside the pods.
# assume we mount `~/test_data` to `/testingdata` in pods.
k8s_volumes = {
    "data": {
        "type": "hostPath",
        "field": {
          "path": os.path.expanduser("~/test_data/"),
          "type": "Directory"
        },
        "mounts": {
          "mountPath": "/testingdata"
        }
    }
}
sess = graphscope.session(k8s_volumes=k8s_volumes)

sess = graphscope.session(k8s_volumes=k8s_volumes, k8s_service_type="LoadBalancer")

g = sess.g()
g = g.add_vertices("/testingdata/ogbn_mag_small/paper.csv", label="paper")
g = g.add_vertices("/testingdata/ogbn_mag_small/author.csv", label="author")
g = g.add_vertices("/testingdata/ogbn_mag_small/institution.csv", label="institution")
g = g.add_vertices("/testingdata/ogbn_mag_small/field_of_study.csv", label="field_of_study")
g = g.add_edges(
    "/testingdata/ogbn_mag_small/author_affiliated_with_institution.csv",
    label="affiliated", src_label="author", dst_label="institution",
)
g = g.add_edges(
    "/testingdata/ogbn_mag_small/paper_has_topic_field_of_study.csv",
    label="hasTopic", src_label="paper", dst_label="field_of_study",
)
g = g.add_edges(
    "/testingdata/ogbn_mag_small/paper_cites_paper.csv",
    label="cites", src_label="paper", dst_label="paper",
)
g = g.add_edges(
    "/testingdata/ogbn_mag_small/author_writes_paper.csv",
    label="writes", src_label="author", dst_label="paper",
)

from graphscope.dataset.ogbn_mag import load_ogbn_mag
g = load_ogbn_mag(sess, "/testingdata/ogbn_mag_small/")

# get the endpoint for submitting Gremlin queries on graph g.
interactive = sess.gremlin(g)
# count the number of papers two authors (with id 2 and 4307) have co-authored
papers = interactive.execute("g.V().has('author', 'id', 2).out('writes').where(__.in('writes').has('id', 4307)).count()").one()

# extract a subgraph of publication within a time range
sub_graph = interactive.subgraph("g.V().has('year', inside(2014, 2020)).outE('cites')")
# project the projected graph to simple graph.
simple_g = sub_graph.project(vertices={"paper": []}, edges={"cites": []})
ret1 = graphscope.k_core(simple_g, k=5)
ret2 = graphscope.triangles(simple_g)
# add the results as new columns to the citation graph
sub_graph = sub_graph.add_column(ret1, {"kcore": "r"})
sub_graph = sub_graph.add_column(ret2, {"tc": "r"})

# define the features for learning
paper_features = []
for i in range(128):
    paper_features.append("feat_" + str(i))
paper_features.append("kcore")
paper_features.append("tc")
# launch a learning engine.
lg = sess.learning(sub_graph, nodes=[("paper", paper_features)],
                  edges=[("paper", "cites", "paper")],
                  gen_labels=[
                      ("train", "paper", 100, (0, 75)),
                      ("val", "paper", 100, (75, 85)),
                      ("test", "paper", 100, (85, 100))
                  ])

# Note: Here we use tensorflow as NN backend to train GNN model. so please
# install tensorflow.
from graphscope.learning.examples import GCN
from graphscope.learning.graphlearn.python.model.tf.trainer import LocalTFTrainer
from graphscope.learning.graphlearn.python.model.tf.optimizer import get_tf_optimizer
# supervised GCN.
def train(config, graph):
    def model_fn():
        return GCN(
            graph,
            config["class_num"],
            config["features_num"],
            config["batch_size"],
            val_batch_size=config["val_batch_size"],
            test_batch_size=config["test_batch_size"],
            categorical_attrs_desc=config["categorical_attrs_desc"],
            hidden_dim=config["hidden_dim"],
            in_drop_rate=config["in_drop_rate"],
            neighs_num=config["neighs_num"],
            hops_num=config["hops_num"],
            node_type=config["node_type"],
            edge_type=config["edge_type"],
            full_graph_mode=config["full_graph_mode"],
        )
    trainer = LocalTFTrainer(
        model_fn,
        epoch=config["epoch"],
        optimizer=get_tf_optimizer(
            config["learning_algo"], config["learning_rate"], config["weight_decay"]
        ),
    )
    trainer.train_and_evaluate()
config = {
    "class_num": 349,  # output dimension
    "features_num": 130,  # 128 dimension + kcore + triangle count
    "batch_size": 500,
    "val_batch_size": 100,
    "test_batch_size": 100,
    "categorical_attrs_desc": "",
    "hidden_dim": 256,
    "in_drop_rate": 0.5,
    "hops_num": 2,
    "neighs_num": [5, 10],
    "full_graph_mode": False,
    "agg_type": "gcn",  # mean, sum
    "learning_algo": "adam",
    "learning_rate": 0.0005,
    "weight_decay": 0.000005,
    "epoch": 20,
    "node_type": "paper",
    "edge_type": "cites",
}
train(config, lg)

sess.close()

# for the first time, run this script to install make, doxygen for docs and java env for testing.
# ./scripts/prepare_dev.sh
make graphscope
make interactive_manager
# by default, the built image is tagged as graphscope/graphscope:SHORTSHA and graphscope/maxgraph_standalone_manager:SHORTSHA

cd python
python3 setup.py install

# for the first time, run this script to install make, doxygen for docs and java env for testing.
# ./scripts/prepare_dev.sh
# run all test cases
./scripts/test.sh --all
# run all test cases on your built image
./scripts/test.sh --all --gs_image graphscope/graphscope:SHORTSHA --gie_manager_image graphscope/maxgraph_standalone_manager:SHORTSHA
# or run the selected cases on a certain module. e.g.,
./scripts/test.sh --python
./scripts/test.sh --gie

# build the docs
make graphscope-docs
# to open preview on local
open docs/_build/html/index.html