From f8a23c89805dddcdf8b6bed8ce3698c90dfd8c9c Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Mon, 22 Sep 2014 13:17:22 -0400 Subject: [PATCH 1/5] Start adding more documentation --- .../web/controller/AbstractController.java | 4 + .../web/controller/BranchController.java | 104 +++++++++++++++++- .../controller/ElasticSearchController.java | 38 ++++++- .../web/controller/GraphController.java | 56 ++++++++++ .../web/controller/GraphExportController.java | 3 + .../web/controller/GraphImportController.java | 4 + .../web/controller/JobController.java | 3 + .../web/controller/ProjectController.java | 3 + .../web/controller/UserController.java | 3 + 9 files changed, 213 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/lab41/dendrite/web/controller/AbstractController.java b/src/main/java/org/lab41/dendrite/web/controller/AbstractController.java index b47828f..364a077 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/AbstractController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/AbstractController.java @@ -29,6 +29,10 @@ import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseStatus; +/** + * The abstract class {@code AbstractController} is the superclass of all controllers. + * It provides error handling when known exceptions. + */ @Controller public abstract class AbstractController { diff --git a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java index f3beedd..ff68e09 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java @@ -27,6 +27,9 @@ import java.security.Principal; import java.util.*; +/** + * The {@code BranchController} implements the user interface for manipulating a graph's branches. + */ @Controller @RequestMapping("/api") public class BranchController extends AbstractController { @@ -34,6 +37,12 @@ public class BranchController extends AbstractController { @Autowired HistoryService historyService; + /** + * Return all branches tracked by dendrite that this user is allowed to see. + * + * @param principal the authentication principal. + * @return the response. + */ // Note this doesn't use @PreAuthorize on purpose because it'll only show the user's projects. @RequestMapping(value = "/branches", method = RequestMethod.GET) @ResponseBody @@ -65,6 +74,13 @@ public GetBranchesResponse getBranches(Principal principal) { return getBranchesResponse; } + /** + * Get a specific branch if this user is allowed to see this branch. + * + * @param branchId the id of the branch + * @return The branch response. + * @throws NotFound + */ @PreAuthorize("hasPermission(#branchId, 'branch', 'admin')") @RequestMapping(value = "/branches/{branchId}", method = RequestMethod.GET) @ResponseBody @@ -84,6 +100,16 @@ public GetBranchResponse getBranch(@PathVariable String branchId) throws NotFoun } } + /** + * Delete a branch. + * + * @param branchId the id of the branch. + * @return The delete branch response. + * @throws IOException + * @throws GitAPIException + * @throws CannotDeleteCurrentBranchException + * @throws NotFound + */ @PreAuthorize("hasPermission(#branchId, 'branch', 'admin')") @RequestMapping(value = "/branches/{branchId}", method = RequestMethod.DELETE) @ResponseBody @@ -126,6 +152,13 @@ public DeleteBranchResponse deleteBranch(@PathVariable String branchId) throws I return new DeleteBranchResponse(); } + /** + * Get all the branches in a project. + * + * @param projectId the project id + * @return the branches + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/branches", method = RequestMethod.GET) @ResponseBody @@ -150,6 +183,14 @@ public GetBranchesResponse getBranches(@PathVariable String projectId) throws No } } + /** + * Get a branch named {@code branchName} in the project. + * + * @param projectId the project id + * @param branchName the branch name + * @return the branch + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/branches/{branchName}", method = RequestMethod.GET) @ResponseBody @@ -174,19 +215,32 @@ public GetBranchResponse getBranch(@PathVariable String projectId, @PathVariable } } + /** + * Create a branch named @{code branchName} in the project. + * + * @param projectId The id of the project. + * @param branchName The name of the branch. + * @param createBranchRequest The branch creation metadata. + * @param result + * @return the branch. + * @throws GitAPIException + * @throws IOException + * @throws BindingException + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/branches/{branchName}", method = RequestMethod.PUT) @ResponseBody public GetBranchResponse createBranch(@PathVariable String projectId, @PathVariable String branchName, - @Valid @RequestBody CreateBranchRequest item, + @Valid @RequestBody CreateBranchRequest createBranchRequest, BindingResult result) throws GitAPIException, IOException, BindingException, NotFound { if (result.hasErrors()) { throw new BindingException(result); } - String graphId = item.getGraphId(); + String graphId = createBranchRequest.getGraphId(); MetaGraphTx tx = metaGraphService.newTransaction(); BranchCommitJob branchCommitJob; @@ -253,6 +307,13 @@ public GetBranchResponse createBranch(@PathVariable String projectId, return getBranchResponse; } + /** + * Get the current branch of a project. + * + * @param projectId the project id + * @return the branch + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/current-branch", method = RequestMethod.GET) @ResponseBody @@ -277,6 +338,18 @@ public GetBranchResponse getCurrentBranch(@PathVariable String projectId) throws } } + /** + * Set the current project's branch. + * + * @param projectId the project id + * @param item + * @param result + * @return + * @throws GitAPIException + * @throws IOException + * @throws BindingException + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/current-branch", method = RequestMethod.PUT) @ResponseBody @@ -324,6 +397,14 @@ public SetCurrentBranchResponse setCurrentBranch(@PathVariable String projectId, return new SetCurrentBranchResponse(); } + /** + * Commit the current branch. + * @param projectId + * @return + * @throws GitAPIException + * @throws IOException + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/current-branch/commit", method = RequestMethod.POST) @ResponseBody @@ -430,6 +511,15 @@ public ResponseEntity> commitBranch(@PathVariable String pro } */ + /** + * Commit a subset of the graph. + * @param projectId + * @param item + * @param result + * @return + * @throws BindingException + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/current-branch/commit-subset", method = RequestMethod.POST) @ResponseBody @@ -490,6 +580,16 @@ public BranchJobResponse commitSubsetBranch(@PathVariable String projectId, return new BranchJobResponse(branchCommitSubsetJob); } + /** + * Export a subset of the project into a new project. + * @param projectId + * @param item + * @param principal + * @param result + * @return + * @throws NotFound + * @throws BindingException + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/current-branch/export-subset", method = RequestMethod.POST) @ResponseBody diff --git a/src/main/java/org/lab41/dendrite/web/controller/ElasticSearchController.java b/src/main/java/org/lab41/dendrite/web/controller/ElasticSearchController.java index 77350fe..d594d81 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/ElasticSearchController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/ElasticSearchController.java @@ -55,6 +55,15 @@ public class ElasticSearchController extends AbstractController { @Autowired MetaGraphService metaGraphService; + /** + * Search a branch. + * + * @param branchId + * @param body + * @return + * @throws JSONException + * @throws UnsupportedEncodingException + */ @PreAuthorize("hasPermission(#branchId, 'branch', 'admin')") @RequestMapping(value = "/api/branches/{branchId}/search", method = RequestMethod.POST) public ResponseEntity branchSearch(@PathVariable String branchId, @@ -93,6 +102,15 @@ public ResponseEntity branchSearch(@PathVariable String branchId, } } + /** + * Search a graph. + * + * @param graphId + * @param body + * @return + * @throws JSONException + * @throws UnsupportedEncodingException + */ @PreAuthorize("hasPermission(#graphId, 'graph', 'admin')") @RequestMapping(value = "/api/graphs/{graphId}/search", method = RequestMethod.POST) public ResponseEntity graphSearch(@PathVariable String graphId, @@ -127,7 +145,14 @@ public ResponseEntity graphSearch(@PathVariable String graphId, return new ResponseEntity<>(response.toString(), responseHeaders, HttpStatus.OK); } - + /** + * Return the index mapping for a branch. + * + * @param branchId + * @return + * @throws JSONException + * @throws IOException + */ @PreAuthorize("hasPermission(#branchId, 'branch', 'admin')") @RequestMapping(value = "/api/branches/{branchId}/search/mapping", method = RequestMethod.GET) public ResponseEntity> branchMapping(@PathVariable String branchId) throws JSONException, IOException { @@ -159,7 +184,14 @@ public ResponseEntity> branchMapping(@PathVariable String br } } - + /** + * Get the index mapping for the graph. + * + * @param graphId + * @return + * @throws JSONException + * @throws IOException + */ @PreAuthorize("hasPermission(#graphId, 'graph', 'admin')") @RequestMapping(value = "/api/graphs/{graphId}/search/mapping", method = RequestMethod.GET) public ResponseEntity> graphMapping(@PathVariable String graphId) throws JSONException, IOException { @@ -167,7 +199,7 @@ public ResponseEntity> graphMapping(@PathVariable String gra } - public ResponseEntity> graphMapping(GraphMetadata.Id graphId) throws JSONException, IOException { + private ResponseEntity> graphMapping(GraphMetadata.Id graphId) throws JSONException, IOException { Map response = new HashMap<>(); diff --git a/src/main/java/org/lab41/dendrite/web/controller/GraphController.java b/src/main/java/org/lab41/dendrite/web/controller/GraphController.java index 685d1a6..64d5d3b 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/GraphController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/GraphController.java @@ -28,6 +28,9 @@ import java.security.Principal; import java.util.*; +/** + * The {@code GraphController} represents the user interface for accessing the metadata about a graph. + */ @Controller @RequestMapping("/api") public class GraphController extends AbstractController { @@ -35,6 +38,12 @@ public class GraphController extends AbstractController { @Autowired MetaGraphService metaGraphService; + /** + * Return all the graphs. + * + * @param principal + * @return + */ // Note this doesn't use @PreAuthorize on purpose because it'll only show the user's graphs. @RequestMapping(value = "/graphs", method = RequestMethod.GET) @ResponseBody @@ -63,6 +72,13 @@ public GetGraphsResponse getGraphs(Principal principal) { return new GetGraphsResponse(graphs); } + /** + * Return a specific graph. + * + * @param graphId + * @return + * @throws NotFound + */ @PreAuthorize("hasPermission(#graphId, 'graph', 'admin')") @RequestMapping(value = "/graphs/{graphId}", method = RequestMethod.GET) @ResponseBody @@ -85,6 +101,14 @@ public GetGraphResponse getGraph(@PathVariable String graphId) throws NotFound { } } + /** + * Return a random subset of nodes and edges of a graph. This is used to render a + * subset of the graph in the UI. + * + * @param graphId + * @return + * @throws NotFound + */ @PreAuthorize("hasPermission(#graphId, 'graph', 'admin')") @RequestMapping(value = "/graphs/{graphId}/random", method = RequestMethod.GET) public ResponseEntity> getRandom(@PathVariable String graphId) throws NotFound { @@ -160,6 +184,13 @@ private void addEdge(Map verticesMap, Map edgesM } } + /** + * Delete a graph. + * + * @param graphId + * @return + * @throws NotFound + */ @PreAuthorize("hasPermission(#graphId, 'graph', 'admin')") @RequestMapping(value = "/graphs/{graphId}", method = RequestMethod.DELETE) public ResponseEntity> deleteGraph(@PathVariable String graphId) throws NotFound { @@ -190,6 +221,13 @@ public ResponseEntity> deleteGraph(@PathVariable String grap } } + /** + * Get all of a project's graphs. + * + * @param projectId + * @return + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/graphs", method = RequestMethod.GET) @ResponseBody @@ -214,6 +252,17 @@ public GetGraphsResponse getGraphs(@PathVariable String projectId) throws NotFou } } + /** + * Create a graph in a project. + * + * @param projectId + * @param item + * @param result + * @param builder + * @return + * @throws BindingException + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/graphs", method = RequestMethod.POST) @ResponseBody @@ -252,6 +301,13 @@ public ResponseEntity createGraph(@PathVariable String project return new ResponseEntity<>(getGraphResponse, headers, HttpStatus.CREATED); } + /** + * Return a project's current graph. + * + * @param projectId + * @return + * @throws NotFound + */ @PreAuthorize("hasPermission(#projectId, 'project', 'admin')") @RequestMapping(value = "/projects/{projectId}/current-graph", method = RequestMethod.GET) @ResponseBody diff --git a/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java b/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java index 8d330a8..9dacb55 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/GraphExportController.java @@ -52,6 +52,9 @@ import java.util.HashMap; import java.util.Map; +/** + * The {@code GraphExportController} represents the user interface for exporting a graph into a file format. + */ @Controller public class GraphExportController extends AbstractController { diff --git a/src/main/java/org/lab41/dendrite/web/controller/GraphImportController.java b/src/main/java/org/lab41/dendrite/web/controller/GraphImportController.java index 5cd6bf8..736f171 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/GraphImportController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/GraphImportController.java @@ -51,6 +51,10 @@ import java.io.InputStream; import java.util.*; +/** + * The {@code GraphImportController} represents the user interface for importing a graph from a + * variety of file formats. + */ @Controller public class GraphImportController extends AbstractController { diff --git a/src/main/java/org/lab41/dendrite/web/controller/JobController.java b/src/main/java/org/lab41/dendrite/web/controller/JobController.java index 142d508..63b27ce 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/JobController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/JobController.java @@ -21,6 +21,9 @@ import java.util.ArrayList; import java.util.List; +/** + * The {@code JobController} is the user interface for creating and monitoring jobs. + */ @Controller @RequestMapping("/api") public class JobController extends AbstractController { diff --git a/src/main/java/org/lab41/dendrite/web/controller/ProjectController.java b/src/main/java/org/lab41/dendrite/web/controller/ProjectController.java index af47beb..b653008 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/ProjectController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/ProjectController.java @@ -22,6 +22,9 @@ import java.security.Principal; import java.util.*; +/** + * The {@code ProjectController} is the main user interface for creating and manipulating projects. + */ @Controller @RequestMapping("/api") public class ProjectController extends AbstractController { diff --git a/src/main/java/org/lab41/dendrite/web/controller/UserController.java b/src/main/java/org/lab41/dendrite/web/controller/UserController.java index 8f2ad19..883b739 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/UserController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/UserController.java @@ -36,6 +36,9 @@ import java.util.List; import java.util.Map; +/** + * The {@code UserController} is the main user interface for creating and manipulating users. + */ @Controller @RequestMapping("/api") public class UserController extends AbstractController { From 07c299e263b1d5c30ef30b2cab6a75f2e62fe134 Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Mon, 22 Sep 2014 13:38:33 -0400 Subject: [PATCH 2/5] DendriteRexsterApplication.getGraphNames() should be read/write This is needed to properly create a user. --- .../org/lab41/dendrite/rexster/DendriteRexsterApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/lab41/dendrite/rexster/DendriteRexsterApplication.java b/src/main/java/org/lab41/dendrite/rexster/DendriteRexsterApplication.java index 100022b..4a0353d 100644 --- a/src/main/java/org/lab41/dendrite/rexster/DendriteRexsterApplication.java +++ b/src/main/java/org/lab41/dendrite/rexster/DendriteRexsterApplication.java @@ -109,7 +109,7 @@ public Set getGraphNames() { Set graphNames = new TreeSet<>(); - MetaGraphTx tx = metaGraphService.buildTransaction().readOnly().start(); + MetaGraphTx tx = metaGraphService.buildTransaction().start(); try { UserMetadata userMetadata = tx.getOrCreateUser(authentication); From d118502afdfdaaa8c7ae439cecb7e70338110c5d Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Wed, 24 Sep 2014 14:57:12 -0400 Subject: [PATCH 3/5] Derive AbstractJob from Callable --- .../dendrite/jobs/AbstractGraphCommitJob.java | 8 +++++--- .../dendrite/jobs/AbstractGraphUpdateJob.java | 14 +++++--------- .../java/org/lab41/dendrite/jobs/AbstractJob.java | 4 +++- .../java/org/lab41/dendrite/jobs/FaunusJob.java | 4 ++-- .../services/analysis/EdgeDegreesService.java | 2 +- .../analysis/jung/BarycenterDistanceService.java | 4 ++-- .../jung/BetweennessCentralityService.java | 4 ++-- .../analysis/jung/ClosenessCentralityService.java | 4 ++-- .../jung/EigenvectorCentralityService.java | 4 ++-- .../services/analysis/jung/PageRankService.java | 4 ++-- .../dendrite/web/controller/BranchController.java | 8 ++++---- .../lab41/dendrite/jobs/BranchCommitJobTest.java | 2 +- .../dendrite/jobs/BranchCommitSubsetJobTest.java | 6 +++--- 13 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/lab41/dendrite/jobs/AbstractGraphCommitJob.java b/src/main/java/org/lab41/dendrite/jobs/AbstractGraphCommitJob.java index 42b911a..420e11a 100644 --- a/src/main/java/org/lab41/dendrite/jobs/AbstractGraphCommitJob.java +++ b/src/main/java/org/lab41/dendrite/jobs/AbstractGraphCommitJob.java @@ -20,7 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractGraphCommitJob extends AbstractJob implements Runnable { +public abstract class AbstractGraphCommitJob extends AbstractJob { private Logger logger = LoggerFactory.getLogger(AbstractGraphCommitJob.class); @@ -60,7 +60,7 @@ public GraphMetadata.Id getDstGraphId() { } @Override - public void run() { + public Void call() { logger.debug("Starting commit on " + srcGraphId + " to " @@ -92,10 +92,12 @@ public void run() { } catch (Throwable t) { setJobState(jobId, JobMetadata.ERROR, t.getMessage()); logger.error("error running job: " + t.getMessage()); - return; + return null; } logger.debug("finished job: " + jobId); + + return null; } protected void copyIndices(DendriteGraph srcGraph, DendriteGraph dstGraph) { diff --git a/src/main/java/org/lab41/dendrite/jobs/AbstractGraphUpdateJob.java b/src/main/java/org/lab41/dendrite/jobs/AbstractGraphUpdateJob.java index 5044497..78e2bc8 100644 --- a/src/main/java/org/lab41/dendrite/jobs/AbstractGraphUpdateJob.java +++ b/src/main/java/org/lab41/dendrite/jobs/AbstractGraphUpdateJob.java @@ -1,20 +1,13 @@ package org.lab41.dendrite.jobs; -import com.thinkaurelius.titan.core.TitanTransaction; -import com.thinkaurelius.titan.core.attribute.FullDouble; import com.tinkerpop.blueprints.Vertex; -import com.tinkerpop.blueprints.oupls.jung.GraphJung; -import com.tinkerpop.blueprints.util.wrappers.batch.BatchGraph; -import org.lab41.dendrite.jobs.AbstractJob; import org.lab41.dendrite.metagraph.DendriteGraph; -import org.lab41.dendrite.metagraph.DendriteGraphBatchWrapper; -import org.lab41.dendrite.metagraph.DendriteGraphTx; import org.lab41.dendrite.metagraph.MetaGraph; import org.lab41.dendrite.metagraph.models.JobMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractGraphUpdateJob extends AbstractJob { +public abstract class AbstractGraphUpdateJob extends AbstractJob { static Logger logger = LoggerFactory.getLogger(AbstractGraphUpdateJob.class); @@ -26,7 +19,8 @@ public AbstractGraphUpdateJob(MetaGraph metaGraph, JobMetadata.Id jobId, Dendrit this.graph = graph; } - public void run() { + @Override + public Void call() throws Exception { logger.debug("Starting " + getClass().getSimpleName() + " analysis on " + graph.getId() + " job " + jobId @@ -59,6 +53,8 @@ public void run() { } logger.debug("Finished analysis on " + jobId); + + return null; } protected abstract void updateGraph(); diff --git a/src/main/java/org/lab41/dendrite/jobs/AbstractJob.java b/src/main/java/org/lab41/dendrite/jobs/AbstractJob.java index 61e2b9a..44eee6f 100644 --- a/src/main/java/org/lab41/dendrite/jobs/AbstractJob.java +++ b/src/main/java/org/lab41/dendrite/jobs/AbstractJob.java @@ -8,7 +8,9 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractJob { +import java.util.concurrent.Callable; + +public abstract class AbstractJob implements Callable { static Logger logger = LoggerFactory.getLogger(AbstractJob.class); diff --git a/src/main/java/org/lab41/dendrite/jobs/FaunusJob.java b/src/main/java/org/lab41/dendrite/jobs/FaunusJob.java index 6c187c7..2728447 100644 --- a/src/main/java/org/lab41/dendrite/jobs/FaunusJob.java +++ b/src/main/java/org/lab41/dendrite/jobs/FaunusJob.java @@ -16,7 +16,7 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; -public class FaunusJob extends AbstractJob implements Callable { +public class FaunusJob extends AbstractJob { private enum State { ACTIVE, DONE, ERROR } @@ -34,7 +34,7 @@ public FaunusJob(MetaGraph metaGraph, JobMetadata.Id jobId, FaunusPipeline faunu } @Override - public Object call() throws Exception { + public Void call() throws Exception { FaunusCompiler compiler = faunusPipeline.getCompiler(); FaunusJobControl jobControl = new FaunusJobControl(faunusPipeline.getGraph(), compiler.getJobs()); diff --git a/src/main/java/org/lab41/dendrite/services/analysis/EdgeDegreesService.java b/src/main/java/org/lab41/dendrite/services/analysis/EdgeDegreesService.java index 8646396..b76554d 100644 --- a/src/main/java/org/lab41/dendrite/services/analysis/EdgeDegreesService.java +++ b/src/main/java/org/lab41/dendrite/services/analysis/EdgeDegreesService.java @@ -53,7 +53,7 @@ public void titanCountDegrees(DendriteGraph graph, JobMetadata.Id jobId) throws jobId, graph); - job.run(); + job.call(); } @Async diff --git a/src/main/java/org/lab41/dendrite/services/analysis/jung/BarycenterDistanceService.java b/src/main/java/org/lab41/dendrite/services/analysis/jung/BarycenterDistanceService.java index 6218997..37e0fff 100644 --- a/src/main/java/org/lab41/dendrite/services/analysis/jung/BarycenterDistanceService.java +++ b/src/main/java/org/lab41/dendrite/services/analysis/jung/BarycenterDistanceService.java @@ -20,12 +20,12 @@ public class BarycenterDistanceService extends AnalysisService { MetaGraphService metaGraphService; @Async - public void run(DendriteGraph graph, JobMetadata.Id jobId) { + public void run(DendriteGraph graph, JobMetadata.Id jobId) throws Exception { BarycenterDistanceJob job = new BarycenterDistanceJob( metaGraphService.getMetaGraph(), jobId, graph); - job.run(); + job.call(); } } diff --git a/src/main/java/org/lab41/dendrite/services/analysis/jung/BetweennessCentralityService.java b/src/main/java/org/lab41/dendrite/services/analysis/jung/BetweennessCentralityService.java index 93088e7..8fa1484 100644 --- a/src/main/java/org/lab41/dendrite/services/analysis/jung/BetweennessCentralityService.java +++ b/src/main/java/org/lab41/dendrite/services/analysis/jung/BetweennessCentralityService.java @@ -20,12 +20,12 @@ public class BetweennessCentralityService extends AnalysisService { MetaGraphService metaGraphService; @Async - public void run(DendriteGraph graph, JobMetadata.Id jobId) { + public void run(DendriteGraph graph, JobMetadata.Id jobId) throws Exception { BetweennessCentralityJob job = new BetweennessCentralityJob( metaGraphService.getMetaGraph(), jobId, graph); - job.run(); + job.call(); } } diff --git a/src/main/java/org/lab41/dendrite/services/analysis/jung/ClosenessCentralityService.java b/src/main/java/org/lab41/dendrite/services/analysis/jung/ClosenessCentralityService.java index a14902c..37583c4 100644 --- a/src/main/java/org/lab41/dendrite/services/analysis/jung/ClosenessCentralityService.java +++ b/src/main/java/org/lab41/dendrite/services/analysis/jung/ClosenessCentralityService.java @@ -20,12 +20,12 @@ public class ClosenessCentralityService extends AnalysisService { MetaGraphService metaGraphService; @Async - public void run(DendriteGraph graph, JobMetadata.Id jobId) { + public void run(DendriteGraph graph, JobMetadata.Id jobId) throws Exception { ClosenessCentralityJob job = new ClosenessCentralityJob( metaGraphService.getMetaGraph(), jobId, graph); - job.run(); + job.call(); } } diff --git a/src/main/java/org/lab41/dendrite/services/analysis/jung/EigenvectorCentralityService.java b/src/main/java/org/lab41/dendrite/services/analysis/jung/EigenvectorCentralityService.java index 062b1ff..562082b 100644 --- a/src/main/java/org/lab41/dendrite/services/analysis/jung/EigenvectorCentralityService.java +++ b/src/main/java/org/lab41/dendrite/services/analysis/jung/EigenvectorCentralityService.java @@ -20,12 +20,12 @@ public class EigenvectorCentralityService extends AnalysisService { MetaGraphService metaGraphService; @Async - public void run(DendriteGraph graph, JobMetadata.Id jobId) { + public void run(DendriteGraph graph, JobMetadata.Id jobId) throws Exception { EigenvectorCentralityJob job = new EigenvectorCentralityJob( metaGraphService.getMetaGraph(), jobId, graph); - job.run(); + job.call(); } } diff --git a/src/main/java/org/lab41/dendrite/services/analysis/jung/PageRankService.java b/src/main/java/org/lab41/dendrite/services/analysis/jung/PageRankService.java index 0512876..722089b 100644 --- a/src/main/java/org/lab41/dendrite/services/analysis/jung/PageRankService.java +++ b/src/main/java/org/lab41/dendrite/services/analysis/jung/PageRankService.java @@ -20,13 +20,13 @@ public class PageRankService extends AnalysisService { MetaGraphService metaGraphService; @Async - public void run(DendriteGraph graph, JobMetadata.Id jobId, double alpha) { + public void run(DendriteGraph graph, JobMetadata.Id jobId, double alpha) throws Exception { PageRankJob job = new PageRankJob( metaGraphService.getMetaGraph(), jobId, graph, alpha); - job.run(); + job.call(); } } diff --git a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java index ff68e09..19ca0a5 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/BranchController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/BranchController.java @@ -302,7 +302,7 @@ public GetBranchResponse createBranch(@PathVariable String projectId, tx.commit(); //taskExecutor.execute(branchCommitJob); - branchCommitJob.run(); + branchCommitJob.call(); return getBranchResponse; } @@ -461,7 +461,7 @@ public BranchJobResponse commitBranch(@PathVariable String projectId) throws Git tx.commit(); //taskExecutor.execute(branchCommitJob); - branchCommitJob.run(); + branchCommitJob.call(); return new BranchJobResponse(branchCommitJob); } @@ -575,7 +575,7 @@ public BranchJobResponse commitSubsetBranch(@PathVariable String projectId, tx.commit(); //taskExecutor.execute(branchCommitSubsetJob); - branchCommitSubsetJob.run(); + branchCommitSubsetJob.call(); return new BranchJobResponse(branchCommitSubsetJob); } @@ -663,7 +663,7 @@ public BranchJobResponse exportSubset(@PathVariable String projectId, tx.commit(); //taskExecutor.execute(branchCommitSubsetJob); - branchCommitSubsetJob.run(); + branchCommitSubsetJob.call(); return new BranchJobResponse(branchCommitSubsetJob); } diff --git a/src/test/java/org/lab41/dendrite/jobs/BranchCommitJobTest.java b/src/test/java/org/lab41/dendrite/jobs/BranchCommitJobTest.java index 5650003..1c09a7b 100644 --- a/src/test/java/org/lab41/dendrite/jobs/BranchCommitJobTest.java +++ b/src/test/java/org/lab41/dendrite/jobs/BranchCommitJobTest.java @@ -65,7 +65,7 @@ public void test() { Assert.assertEquals(srcGraphId, srcGraph.getId()); - branchCommitJob.run(); + branchCommitJob.call(); DendriteGraph dstGraph = metaGraph.getGraph(dstGraphId); Assert.assertNotNull(dstGraph); diff --git a/src/test/java/org/lab41/dendrite/jobs/BranchCommitSubsetJobTest.java b/src/test/java/org/lab41/dendrite/jobs/BranchCommitSubsetJobTest.java index bd8b0b9..3eeb0f3 100644 --- a/src/test/java/org/lab41/dendrite/jobs/BranchCommitSubsetJobTest.java +++ b/src/test/java/org/lab41/dendrite/jobs/BranchCommitSubsetJobTest.java @@ -174,7 +174,7 @@ public void testStep0() { "name:A", 0); - branchCommitSubsetJob.run(); + branchCommitSubsetJob.call(); DendriteGraph dstGraph = metaGraph.getGraph(branchCommitSubsetJob.getDstGraphId()); Assert.assertNotNull(dstGraph); @@ -209,7 +209,7 @@ public void testStep1() { "name:A", 1); - branchCommitSubsetJob.run(); + branchCommitSubsetJob.call(); DendriteGraph dstGraph = metaGraph.getGraph(branchCommitSubsetJob.getDstGraphId()); Assert.assertNotNull(dstGraph); @@ -252,7 +252,7 @@ public void testStep2() { "name:A", 2); - branchCommitSubsetJob.run(); + branchCommitSubsetJob.call(); DendriteGraph dstGraph = metaGraph.getGraph(branchCommitSubsetJob.getDstGraphId()); Assert.assertNotNull(dstGraph); From dc0cbaad20e1997a6ce2a3e6f61a4faae7b7de0e Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Wed, 24 Sep 2014 15:51:28 -0400 Subject: [PATCH 4/5] Start migrating Snap over into a Job --- .../dendrite/jobs/AbstractGraphUpdateJob.java | 2 +- .../dendrite/jobs/ExportEdgeListJob.java | 75 +++++ .../dendrite/jobs/snap/AbstractSnapJob.java | 113 +++++++ .../dendrite/jobs/snap/CentralityJob.java | 99 ++++++ .../dendrite/services/HistoryService.java | 2 - .../services/analysis/SnapService.java | 288 ++---------------- .../controller/analysis/SnapController.java | 12 +- src/main/webapp/WEB-INF/dev/snap.properties | 2 +- src/main/webapp/WEB-INF/prod/snap.properties | 2 +- 9 files changed, 314 insertions(+), 281 deletions(-) create mode 100644 src/main/java/org/lab41/dendrite/jobs/ExportEdgeListJob.java create mode 100644 src/main/java/org/lab41/dendrite/jobs/snap/AbstractSnapJob.java create mode 100644 src/main/java/org/lab41/dendrite/jobs/snap/CentralityJob.java diff --git a/src/main/java/org/lab41/dendrite/jobs/AbstractGraphUpdateJob.java b/src/main/java/org/lab41/dendrite/jobs/AbstractGraphUpdateJob.java index 78e2bc8..02af888 100644 --- a/src/main/java/org/lab41/dendrite/jobs/AbstractGraphUpdateJob.java +++ b/src/main/java/org/lab41/dendrite/jobs/AbstractGraphUpdateJob.java @@ -57,7 +57,7 @@ public Void call() throws Exception { return null; } - protected abstract void updateGraph(); + protected abstract void updateGraph() throws Exception; protected void createIndices() { } diff --git a/src/main/java/org/lab41/dendrite/jobs/ExportEdgeListJob.java b/src/main/java/org/lab41/dendrite/jobs/ExportEdgeListJob.java new file mode 100644 index 0000000..21ac85b --- /dev/null +++ b/src/main/java/org/lab41/dendrite/jobs/ExportEdgeListJob.java @@ -0,0 +1,75 @@ +/* + * Copyright 2014 In-Q-Tel/Lab41 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.lab41.dendrite.jobs; + +import com.tinkerpop.blueprints.Direction; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Vertex; +import org.lab41.dendrite.metagraph.DendriteGraph; +import org.lab41.dendrite.metagraph.MetaGraph; +import org.lab41.dendrite.metagraph.models.JobMetadata; + +import java.io.BufferedOutputStream; +import java.io.OutputStream; +import java.io.UnsupportedEncodingException; + +public class ExportEdgeListJob extends AbstractJob { + + private static final String UTF8 = "UTF-8"; + private static final byte[] TAB; + private static final byte[] NEWLINE; + + static { + try { + TAB = "\t".getBytes(UTF8); + NEWLINE = "\n".getBytes(UTF8); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException("Can not find " + UTF8 + " encoding"); + } + } + + private final DendriteGraph graph; + private final OutputStream outputStream; + + public ExportEdgeListJob(MetaGraph metaGraph, JobMetadata.Id jobId, DendriteGraph graph, OutputStream outputStream) { + super(metaGraph, jobId); + + this.graph = graph; + this.outputStream = outputStream; + } + + @Override + public Void call() throws Exception { + + try (BufferedOutputStream bos = new BufferedOutputStream(outputStream)) { + for (Edge edge : graph.getEdges()) { + Vertex outVertex = edge.getVertex(Direction.OUT); + Vertex inVertex = edge.getVertex(Direction.IN); + String label = edge.getLabel(); + + bos.write(outVertex.getId().toString().getBytes(UTF8)); + bos.write(TAB); + bos.write(inVertex.getId().toString().getBytes(UTF8)); + bos.write(TAB); + bos.write(label.getBytes(UTF8)); + bos.write(NEWLINE); + } + } + + return null; + } +} diff --git a/src/main/java/org/lab41/dendrite/jobs/snap/AbstractSnapJob.java b/src/main/java/org/lab41/dendrite/jobs/snap/AbstractSnapJob.java new file mode 100644 index 0000000..bec97d6 --- /dev/null +++ b/src/main/java/org/lab41/dendrite/jobs/snap/AbstractSnapJob.java @@ -0,0 +1,113 @@ +/* + * Copyright 2014 In-Q-Tel/Lab41 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.lab41.dendrite.jobs.snap; + +import org.apache.commons.io.IOUtils; +import org.lab41.dendrite.jobs.AbstractGraphUpdateJob; +import org.lab41.dendrite.jobs.ExportEdgeListJob; +import org.lab41.dendrite.metagraph.DendriteGraph; +import org.lab41.dendrite.metagraph.MetaGraph; +import org.lab41.dendrite.metagraph.models.JobMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; + +public abstract class AbstractSnapJob extends AbstractGraphUpdateJob { + + private static final Logger logger = LoggerFactory.getLogger(AbstractSnapJob.class); + + protected AbstractSnapJob(MetaGraph metaGraph, JobMetadata.Id jobId, DendriteGraph graph) { + super(metaGraph, jobId, graph); + } + + @Override + protected void updateGraph() throws Exception { + Path tmpDir = Files.createTempDirectory("tmp"); + + try { + Path inputPath = tmpDir.resolve("input"); + + try { + runExport(inputPath); + + Path outputPath = tmpDir.resolve("output"); + + try { + runSnap(inputPath, outputPath); + + runImport(outputPath); + } finally { + Files.delete(outputPath); + } + + } finally { + Files.delete(inputPath); + } + + } finally { + Files.delete(tmpDir); + } + } + + private void runExport(Path exportPath) throws Exception { + try (OutputStream os = new FileOutputStream(exportPath.toFile())) { + ExportEdgeListJob exportEdgeListJob = new ExportEdgeListJob(metaGraph, jobId, graph, os); + exportEdgeListJob.call(); + } + } + + private void runSnap(Path inputFile, Path outputFile) throws Exception { + String cmd = getSnapCommand(inputFile, outputFile); + + logger.debug("running: " + cmd); + + Process p = Runtime.getRuntime().exec(new String[]{"bash", "-c", cmd}); + + int exitStatus = p.waitFor(); + + logger.debug("snap finished with ", exitStatus); + + if (exitStatus != 0) { + String stdout = IOUtils.toString(p.getInputStream()); + String stderr = IOUtils.toString(p.getErrorStream()); + + throw new Exception("Snap process failed: [" + exitStatus + "]:\n" + stdout + "\n" + stderr); + } + } + + protected abstract String getSnapCommand(Path inputFile, Path outputFile); + + private void runImport(Path outputFile) throws IOException { + BufferedReader br = new BufferedReader(new FileReader(outputFile.toFile())); + String line; + + while ((line = br.readLine()) != null) { + // skip over any comments. + if (line.startsWith("#")) { + continue; + } + + String[] parts = line.split("\t"); + importLine(parts); + } + } + + protected abstract void importLine(String[] parts); +} diff --git a/src/main/java/org/lab41/dendrite/jobs/snap/CentralityJob.java b/src/main/java/org/lab41/dendrite/jobs/snap/CentralityJob.java new file mode 100644 index 0000000..98d7af0 --- /dev/null +++ b/src/main/java/org/lab41/dendrite/jobs/snap/CentralityJob.java @@ -0,0 +1,99 @@ +/* + * Copyright 2014 In-Q-Tel/Lab41 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.lab41.dendrite.jobs.snap; + +import com.thinkaurelius.titan.core.attribute.FullDouble; +import com.tinkerpop.blueprints.Vertex; +import org.lab41.dendrite.metagraph.DendriteGraph; +import org.lab41.dendrite.metagraph.MetaGraph; +import org.lab41.dendrite.metagraph.models.JobMetadata; + +import java.nio.file.Path; + +public class CentralityJob extends AbstractSnapJob { + + private static final String DEGREE_KEY = "snapDegrees"; + private static final String CLOSENESS_KEY = "snapCloseness"; + private static final String BETWEENNESS_KEY = "snapBetweenness"; + private static final String EIGENVECTOR_KEY = "snapEigenvector"; + private static final String NETWORK_CONSTRAINT_KEY = "snapNetworkConstraint"; + private static final String CLUSTERING_COEFFICIENT_KEY = "snapClusteringCoefficient"; + private static final String PAGERANK_KEY = "snapPageRank"; + private static final String HUB_SCORE_KEY = "snapHupScore"; + private static final String AUTHORITY_SCORE_KEY = "snapAuthorityScore"; + + private final Path pathToExecutable; + + public CentralityJob(MetaGraph metaGraph, + JobMetadata.Id jobId, + DendriteGraph graph, + Path pathToSnap) { + super(metaGraph, jobId, graph); + + this.pathToExecutable = pathToSnap.resolve("centrality").resolve("centrality"); + } + + @Override + protected String getSnapCommand(Path inputFile, Path outputFile) { + return pathToExecutable.toString() + + " -i:" + inputFile + + " -o:" + outputFile; + } + + @Override + protected void importLine(String[] parts) { + String id = parts[0]; + double degree = Double.valueOf(parts[1]); + double closeness = Double.valueOf(parts[2]); + double betweenness = Double.valueOf(parts[3]); + double eigenvector = Double.valueOf(parts[4]); + double networkConstraint = Double.valueOf(parts[5]); + double clusteringCoefficient = Double.valueOf(parts[6]); + double pagerank = Double.valueOf(parts[7]); + double hubScore = Double.valueOf(parts[8]); + double authorityScore = Double.valueOf(parts[9]); + + // feed snap output as input for updating each vertex + Vertex vertex = graph.getVertex(id); + vertex.setProperty(DEGREE_KEY, degree); + vertex.setProperty(CLOSENESS_KEY, closeness); + vertex.setProperty(BETWEENNESS_KEY, betweenness); + vertex.setProperty(EIGENVECTOR_KEY, eigenvector); + vertex.setProperty(NETWORK_CONSTRAINT_KEY, networkConstraint); + vertex.setProperty(CLUSTERING_COEFFICIENT_KEY, clusteringCoefficient); + vertex.setProperty(PAGERANK_KEY, pagerank); + vertex.setProperty(HUB_SCORE_KEY, hubScore); + vertex.setProperty(AUTHORITY_SCORE_KEY, authorityScore); + } + + @Override + protected void createIndices() { + createVertexIndex(DEGREE_KEY, FullDouble.class); + createVertexIndex(CLOSENESS_KEY, FullDouble.class); + createVertexIndex(BETWEENNESS_KEY, FullDouble.class); + createVertexIndex(EIGENVECTOR_KEY, FullDouble.class); + createVertexIndex(NETWORK_CONSTRAINT_KEY, FullDouble.class); + createVertexIndex(CLUSTERING_COEFFICIENT_KEY, FullDouble.class); + createVertexIndex(PAGERANK_KEY, FullDouble.class); + createVertexIndex(HUB_SCORE_KEY, FullDouble.class); + createVertexIndex(AUTHORITY_SCORE_KEY, FullDouble.class); + createVertexIndex(DEGREE_KEY, FullDouble.class); + createVertexIndex(DEGREE_KEY, FullDouble.class); + createVertexIndex(DEGREE_KEY, FullDouble.class); + createVertexIndex(DEGREE_KEY, FullDouble.class); + } +} diff --git a/src/main/java/org/lab41/dendrite/services/HistoryService.java b/src/main/java/org/lab41/dendrite/services/HistoryService.java index 1140d48..4990530 100644 --- a/src/main/java/org/lab41/dendrite/services/HistoryService.java +++ b/src/main/java/org/lab41/dendrite/services/HistoryService.java @@ -5,7 +5,6 @@ import org.apache.commons.configuration.PropertiesConfiguration; import org.eclipse.jgit.api.Git; import org.eclipse.jgit.api.errors.GitAPIException; -import org.lab41.dendrite.metagraph.DendriteGraph; import org.lab41.dendrite.metagraph.models.ProjectMetadata; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -16,7 +15,6 @@ import org.springframework.stereotype.Service; import java.io.File; - import java.io.IOException; @Service diff --git a/src/main/java/org/lab41/dendrite/services/analysis/SnapService.java b/src/main/java/org/lab41/dendrite/services/analysis/SnapService.java index f6c3623..96c0f84 100644 --- a/src/main/java/org/lab41/dendrite/services/analysis/SnapService.java +++ b/src/main/java/org/lab41/dendrite/services/analysis/SnapService.java @@ -1,22 +1,9 @@ package org.lab41.dendrite.services.analysis; -import com.thinkaurelius.faunus.FaunusGraph; -import com.thinkaurelius.faunus.FaunusPipeline; -import com.thinkaurelius.faunus.formats.edgelist.EdgeListOutputFormat; -import com.thinkaurelius.faunus.formats.titan.hbase.TitanHBaseInputFormat; -import com.thinkaurelius.titan.core.TitanTransaction; -import com.thinkaurelius.titan.core.attribute.FullDouble; -import com.tinkerpop.blueprints.Vertex; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.io.IOUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.permission.FsAction; -import org.apache.hadoop.fs.permission.FsPermission; -import org.lab41.dendrite.jobs.FaunusJob; +import org.lab41.dendrite.jobs.snap.CentralityJob; import org.lab41.dendrite.metagraph.DendriteGraph; import org.lab41.dendrite.metagraph.models.JobMetadata; import org.lab41.dendrite.services.MetaGraphService; @@ -29,274 +16,41 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; -import java.io.*; -import java.net.URI; -import java.util.*; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; @Service public class SnapService extends AnalysisService { Logger logger = LoggerFactory.getLogger(SnapService.class); - private org.apache.commons.configuration.Configuration config; - private static List algorithms = Arrays.asList( - "centrality" - ); + @Value("${snap.properties}") + String pathToProperties; @Autowired ResourceLoader resourceLoader; - @Autowired - MetaGraphService metaGraphService; - - @Autowired - FaunusPipelineService faunusPipelineService; - - @Value("${snap.properties}") - String pathToProperties; - @Async - public void snapAlgorithm(DendriteGraph graph, String algorithm, JobMetadata.Id jobId) throws Exception { - try { - if (!algorithms.contains(algorithm)) { - throw new Exception("invalid algorithm selected"); - } - - Resource resource = resourceLoader.getResource(pathToProperties); - config = new PropertiesConfiguration(resource.getFile()); - - logger.debug("Starting Snap " - + algorithm + " analysis on " - + graph.getId() - + " job " + jobId - + " " + Thread.currentThread().getName()); - - setJobName(jobId, "snap_"+algorithm); - setJobState(jobId, JobMetadata.RUNNING); - - // Make sure the indices exist. - createIndices(graph, algorithm); - - run(graph, jobId, algorithm); - } catch (Exception e) { - logger.debug("snap-" + algorithm + ": error: ", e); - e.printStackTrace(); - setJobState(jobId, JobMetadata.ERROR, e.getMessage()); - throw e; - } - - setJobState(jobId, JobMetadata.DONE); - - logger.debug("Snap " + algorithm + ": finished job: " + jobId); - } - - private void createIndices(DendriteGraph graph, String algorithm) { - TitanTransaction tx = graph.newTransaction(); - - if (algorithm.equals("centrality")) { - if (tx.getType("snap_degree") == null) { - tx.makeKey("snap_degree") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - if (tx.getType("snap_closeness") == null) { - tx.makeKey("snap_closeness") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - if (tx.getType("snap_betweenness") == null) { - tx.makeKey("snap_betweenness") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - if (tx.getType("snap_eigenvector") == null) { - tx.makeKey("snap_eigenvector") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - if (tx.getType("snap_network_constraint") == null) { - tx.makeKey("snap_network_constraint") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - if (tx.getType("snap_clustering_coefficient") == null) { - tx.makeKey("snap_clustering_coefficient") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - if (tx.getType("snap_pagerank") == null) { - tx.makeKey("snap_pagerank") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - if (tx.getType("snap_hub_score") == null) { - tx.makeKey("snap_hub_score") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - if (tx.getType("snap_authority_score") == null) { - tx.makeKey("snap_authority_score") - .dataType(FullDouble.class) - .indexed(DendriteGraph.INDEX_NAME, Vertex.class) - .make(); - } - } - - tx.commit(); - } - - private void run(DendriteGraph graph, JobMetadata.Id jobId, String algorithm) throws Exception { - logger.debug("starting snap analysis of '" + graph.getId() + "'"); + public void snapCentrality(DendriteGraph graph, JobMetadata.Id jobId) throws Exception { + // load configuration + Resource resource = resourceLoader.getResource(pathToProperties); + Configuration config = new PropertiesConfiguration(resource.getFile()); - FileSystem fs = FileSystem.get(new Configuration()); + Path pathToSnap = Paths.get(config.getString("metagraph.template.snap.algorithm-path")); - // Create the temporary directories. - Path tmpDir = new Path( - new Path(new Path(fs.getHomeDirectory(), "dendrite"), "tmp"), - UUID.randomUUID().toString()); + CentralityJob job = new CentralityJob( + metaGraphService.getMetaGraph(), + jobId, + graph, + pathToSnap); - fs.mkdirs(tmpDir); - fs.setPermission(tmpDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true)); - //fs.deleteOnExit(tmpDir); - try { - Path exportDir = new Path(tmpDir, "export"); - Path importDir = new Path(tmpDir, "import"); - - fs.mkdirs(exportDir); - fs.mkdirs(importDir); - fs.setPermission(importDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL, true)); - - runExport(graph, jobId, exportDir); - runSnap(fs, exportDir, importDir, algorithm); - - // We don't need the export directory at this point. - //fs.delete(exportDir, true); - - runImport(graph, fs, importDir, algorithm); - } finally { - // Clean up after ourselves. - fs.delete(tmpDir, true); - - logger.debug("finished snap analysis of '" + graph.getId() + "'"); - } - } - - private void runExport(DendriteGraph graph, JobMetadata.Id jobId, Path exportDir) throws Exception { - FaunusGraph faunusGraph = new FaunusGraph(); - faunusGraph.setGraphInputFormat(TitanHBaseInputFormat.class); - faunusGraph.setGraphOutputFormat(EdgeListOutputFormat.class); - - faunusPipelineService.configureGraph(faunusGraph, exportDir, graph); - FaunusPipeline exportPipeline = new FaunusPipeline(faunusGraph); - exportPipeline._(); - - exportPipeline.done(); - FaunusJob faunusJob = new FaunusJob(metaGraphService.getMetaGraph(), jobId, exportPipeline); - faunusJob.call(); - } - - private void runSnap(FileSystem fs, Path exportDir, Path importDir, String algorithm) throws Exception { - URI uriImport = URI.create("file:///tmp/" + UUID.randomUUID().toString()); - URI uriExport = URI.create("file:///tmp/" + UUID.randomUUID().toString()); - Path tmpImportFile = new Path(uriImport); - Path tmpExportFile = new Path(uriExport); - - exportDir = new Path(exportDir, "job-0/part-m-00000"); - importDir = new Path(importDir, "graph"); - - fs.copyToLocalFile(exportDir, tmpExportFile); - - try { - // feed output to snap as input - String cmd = new Path(config.getString("metagraph.template.snap.algorithm-path"), algorithm) + - " -i:" + tmpExportFile.toString().substring(5) + - " -o:" + tmpImportFile.toString().substring(5); - - logger.debug("running: " + cmd); - - Process p = Runtime.getRuntime().exec(new String[]{"bash", "-c", cmd}); - - int exitStatus = p.waitFor(); - - logger.debug("snap finished with ", exitStatus); - - if (exitStatus != 0) { - String stdout = IOUtils.toString(p.getInputStream()); - String stderr = IOUtils.toString(p.getErrorStream()); - - throw new Exception("Snap process failed: [" + exitStatus + "]:\n" + stdout + "\n" + stderr); - } - fs.copyFromLocalFile(tmpImportFile, importDir); - } finally { - //tmpFile.delete(); - } + job.call(); } - private void runImport(DendriteGraph graph, FileSystem fs, Path importDir, String algorithm) throws IOException { - // FIXME this is due to the AdjacencyFileInputFormat not properly creating edges - TitanTransaction tx = graph.newTransaction(); - - try { - for (FileStatus status: fs.listStatus(importDir)) { - BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(status.getPath()))); - String line; - - // get rid of header - line = br.readLine(); - for (int i = 0; i < 3; i++) { - line = br.readLine(); - } - - while (line != null) { - String[] parts; - parts = line.split("\t"); - - String id = parts[0]; - if (algorithm.equals("centrality")) { - double degree = Double.valueOf(parts[1]); - double closeness = Double.valueOf(parts[2]); - double betweenness = Double.valueOf(parts[3]); - double eigenvector = Double.valueOf(parts[4]); - double networkConstraint = Double.valueOf(parts[5]); - double clusteringCoefficient = Double.valueOf(parts[6]); - double pagerank = Double.valueOf(parts[7]); - double hubScore = Double.valueOf(parts[8]); - double authorityScore = Double.valueOf(parts[9]); - - // feed snap output as input for updating each vertex - Vertex vertex = tx.getVertex(id); - vertex.setProperty("snap_degree", degree); - vertex.setProperty("snap_closeness", closeness); - vertex.setProperty("snap_betweenness", betweenness); - vertex.setProperty("snap_eigenvector", eigenvector); - vertex.setProperty("snap_network_constraint", networkConstraint); - vertex.setProperty("snap_clustering_coefficient", clusteringCoefficient); - vertex.setProperty("snap_pagerank", pagerank); - vertex.setProperty("snap_hub_score", hubScore); - vertex.setProperty("snap_authority_score", authorityScore); - } else { - double value = Double.valueOf(parts[1]); - // feed snap output as input for updating each vertex - Vertex vertex = tx.getVertex(id); - vertex.setProperty("snap_" + algorithm, value); - } - line = br.readLine(); - } - } - } catch (Exception e) { - tx.rollback(); - throw e; - } - tx.commit(); } } diff --git a/src/main/java/org/lab41/dendrite/web/controller/analysis/SnapController.java b/src/main/java/org/lab41/dendrite/web/controller/analysis/SnapController.java index 248186b..7521529 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/analysis/SnapController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/analysis/SnapController.java @@ -30,8 +30,8 @@ public class SnapController extends AbstractController { SnapService snapService; @PreAuthorize("hasPermission(#graphId, 'graph', 'admin')") - @RequestMapping(value = "/api/graphs/{graphId}/analysis/snap/{algorithm}", method = RequestMethod.POST) - public ResponseEntity> startJob(@PathVariable String graphId, @PathVariable String algorithm) throws Exception { + @RequestMapping(value = "/api/graphs/{graphId}/analysis/snap/centrality", method = RequestMethod.POST) + public ResponseEntity> startJob(@PathVariable String graphId) throws Exception { Map response = new HashMap<>(); @@ -68,14 +68,8 @@ public ResponseEntity> startJob(@PathVariable String graphId tx.commit(); - if (!algorithm.equals("centrality")) { - response.put("status", "error"); - response.put("msg", algorithm + " not implemented"); - return new ResponseEntity<>(response, HttpStatus.NOT_FOUND); - } - // We can't pass the values directly because they'll live in a separate thread. - snapService.snapAlgorithm(graph, algorithm, jobMetadata.getId()); + snapService.snapCentrality(graph, jobMetadata.getId()); return new ResponseEntity<>(response, HttpStatus.OK); } diff --git a/src/main/webapp/WEB-INF/dev/snap.properties b/src/main/webapp/WEB-INF/dev/snap.properties index c6c90bf..d8f8374 100644 --- a/src/main/webapp/WEB-INF/dev/snap.properties +++ b/src/main/webapp/WEB-INF/dev/snap.properties @@ -1 +1 @@ -metagraph.template.snap.algorithm-path=/Snap-2.2/examples/centrality/ +metagraph.template.snap.algorithm-path=~/Snap-2.2/examples/ diff --git a/src/main/webapp/WEB-INF/prod/snap.properties b/src/main/webapp/WEB-INF/prod/snap.properties index 47f91c0..d8f8374 100644 --- a/src/main/webapp/WEB-INF/prod/snap.properties +++ b/src/main/webapp/WEB-INF/prod/snap.properties @@ -1 +1 @@ -metagraph.template.snap.algorithm-path=~/Snap-2.2/examples/centrality/ +metagraph.template.snap.algorithm-path=~/Snap-2.2/examples/ From 2f1e99849acdb456daa68837ac4fb400be385ad2 Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Wed, 24 Sep 2014 15:53:01 -0400 Subject: [PATCH 5/5] Add backend support for running snap girvan newman --- .../dendrite/jobs/snap/CommunityJob.java | 64 +++++++++++++++++++ .../services/analysis/SnapService.java | 14 ++++ .../controller/analysis/SnapController.java | 46 ++++++++++++- 3 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/lab41/dendrite/jobs/snap/CommunityJob.java diff --git a/src/main/java/org/lab41/dendrite/jobs/snap/CommunityJob.java b/src/main/java/org/lab41/dendrite/jobs/snap/CommunityJob.java new file mode 100644 index 0000000..9803e94 --- /dev/null +++ b/src/main/java/org/lab41/dendrite/jobs/snap/CommunityJob.java @@ -0,0 +1,64 @@ +/* + * Copyright 2014 In-Q-Tel/Lab41 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.lab41.dendrite.jobs.snap; + +import com.thinkaurelius.titan.core.attribute.FullDouble; +import com.tinkerpop.blueprints.Vertex; +import org.lab41.dendrite.metagraph.DendriteGraph; +import org.lab41.dendrite.metagraph.MetaGraph; +import org.lab41.dendrite.metagraph.models.JobMetadata; + +import java.nio.file.Path; + +public class CommunityJob extends AbstractSnapJob { + + private static final String COMMUNITY_KEY = "snapCommunity"; + + private final Path pathToExecutable; + + public CommunityJob(MetaGraph metaGraph, + JobMetadata.Id jobId, + DendriteGraph graph, + Path pathToSnap) { + super(metaGraph, jobId, graph); + + this.pathToExecutable = pathToSnap.resolve("community").resolve("community"); + } + + @Override + protected String getSnapCommand(Path inputFile, Path outputFile) { + return pathToExecutable.toString() + + " -i:" + inputFile + + " -o:" + outputFile + + " -a:1"; + } + + @Override + protected void importLine(String[] parts) { + String id = parts[0]; + double community = Double.valueOf(parts[1]); + + // feed snap output as input for updating each vertex + Vertex vertex = graph.getVertex(id); + vertex.setProperty(COMMUNITY_KEY, community); + } + + @Override + protected void createIndices() { + createVertexIndex(COMMUNITY_KEY, Integer.class); + } +} diff --git a/src/main/java/org/lab41/dendrite/services/analysis/SnapService.java b/src/main/java/org/lab41/dendrite/services/analysis/SnapService.java index 96c0f84..06da8e9 100644 --- a/src/main/java/org/lab41/dendrite/services/analysis/SnapService.java +++ b/src/main/java/org/lab41/dendrite/services/analysis/SnapService.java @@ -4,6 +4,7 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.lab41.dendrite.jobs.snap.CentralityJob; +import org.lab41.dendrite.jobs.snap.CommunityJob; import org.lab41.dendrite.metagraph.DendriteGraph; import org.lab41.dendrite.metagraph.models.JobMetadata; import org.lab41.dendrite.services.MetaGraphService; @@ -50,7 +51,20 @@ public void snapCentrality(DendriteGraph graph, JobMetadata.Id jobId) throws Exc job.call(); } + @Async + public void snapCommunity(DendriteGraph graph, JobMetadata.Id jobId) throws Exception { + // load configuration + Resource resource = resourceLoader.getResource(pathToProperties); + Configuration config = new PropertiesConfiguration(resource.getFile()); + Path pathToSnap = Paths.get(config.getString("metagraph.template.snap.algorithm-path")); + CommunityJob job = new CommunityJob( + metaGraphService.getMetaGraph(), + jobId, + graph, + pathToSnap); + + job.call(); } } diff --git a/src/main/java/org/lab41/dendrite/web/controller/analysis/SnapController.java b/src/main/java/org/lab41/dendrite/web/controller/analysis/SnapController.java index 7521529..2ef707f 100644 --- a/src/main/java/org/lab41/dendrite/web/controller/analysis/SnapController.java +++ b/src/main/java/org/lab41/dendrite/web/controller/analysis/SnapController.java @@ -31,7 +31,7 @@ public class SnapController extends AbstractController { @PreAuthorize("hasPermission(#graphId, 'graph', 'admin')") @RequestMapping(value = "/api/graphs/{graphId}/analysis/snap/centrality", method = RequestMethod.POST) - public ResponseEntity> startJob(@PathVariable String graphId) throws Exception { + public ResponseEntity> startCentrality(@PathVariable String graphId) throws Exception { Map response = new HashMap<>(); @@ -74,4 +74,48 @@ public ResponseEntity> startJob(@PathVariable String graphId return new ResponseEntity<>(response, HttpStatus.OK); } + @PreAuthorize("hasPermission(#graphId, 'graph', 'admin')") + @RequestMapping(value = "/api/graphs/{graphId}/analysis/snap/community", method = RequestMethod.POST) + public ResponseEntity> startCommunity(@PathVariable String graphId) throws Exception { + + Map response = new HashMap<>(); + + DendriteGraph graph = metaGraphService.getDendriteGraph(graphId); + if (graph == null) { + response.put("status", "error"); + response.put("msg", "missing graph metadata '" + graphId + "'"); + return new ResponseEntity<>(response, HttpStatus.NOT_FOUND); + } + + MetaGraphTx tx = metaGraphService.newTransaction(); + + GraphMetadata graphMetadata = tx.getGraph(graphId); + if (graphMetadata == null) { + response.put("status", "error"); + response.put("msg", "missing graph metadata '" + graphId + "'"); + tx.rollback(); + return new ResponseEntity<>(response, HttpStatus.NOT_FOUND); + } + + ProjectMetadata projectMetadata = graphMetadata.getProject(); + if (projectMetadata == null) { + response.put("status", "error"); + response.put("msg", "missing project metadata for graph '" + graphId + "'"); + tx.rollback(); + return new ResponseEntity<>(response, HttpStatus.NOT_FOUND); + } + + JobMetadata jobMetadata = tx.createJob(projectMetadata); + + response.put("status", "ok"); + response.put("msg", "job submitted"); + response.put("jobId", jobMetadata.getId().toString()); + + tx.commit(); + + // We can't pass the values directly because they'll live in a separate thread. + snapService.snapCommunity(graph, jobMetadata.getId()); + + return new ResponseEntity<>(response, HttpStatus.OK); + } }