diff --git a/src/main/java/org/wikitolearn/wikirating/controller/PageController.java b/src/main/java/org/wikitolearn/wikirating/controller/PageController.java index bda3e30..4bd8425 100644 --- a/src/main/java/org/wikitolearn/wikirating/controller/PageController.java +++ b/src/main/java/org/wikitolearn/wikirating/controller/PageController.java @@ -1,116 +1,112 @@ /** * */ package org.wikitolearn.wikirating.controller; import java.util.ArrayList; import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.RestController; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.model.Vote; -import org.wikitolearn.wikirating.service.mediawiki.PageMediaWikiService; /** * @author aletundo RESTController for pages resources. It handles all the * requests to /pages. */ @RestController @RequestMapping("/pages/") public class PageController { private static final Logger LOG = LoggerFactory.getLogger(PageController.class); - @Autowired - private PageMediaWikiService pageMediaWikiService; + /** * Handle GET requests on /pages/{pageId} URI. The last revision of the page is returned. * * @param pageId * int The id of the page * @return response Revision */ @RequestMapping(value = "{pageId}", method = RequestMethod.GET, produces = "application/json") @ResponseBody public Revision getLastRevisionByPageId(@PathVariable("pageId") int pageId) { // TODO Get the last available revision for the requested page - pageMediaWikiService.getCourseTree("https://it.wikitolearn.vodka/api.php", "Course:Fisica moderna"); return new Revision(); } /** * Handle GET requests on /pages/{pageId}/revisions URI. All the revision for the requested page are returned. * * @param pageId * int The id of the page * @return response The list with all the revisions */ @RequestMapping(value = "{pageId}/revisions", method = RequestMethod.GET, produces = "application/json") @ResponseBody public List getAllPageRevisions(@PathVariable("pageId") int pageId) { // TODO Get the requested revision return new ArrayList(); } /** * Handle GET requests on /pages/{pageId}/revisions/{revId} URI. The requested page revision is returned. * * @param pageId * int The id of the page * @param revId * int The id of the revision * @return response Revision */ @RequestMapping(value = "{pageId}/revisions/{revId}", method = RequestMethod.GET, produces = "application/json") @ResponseBody public Revision getRevisionById(@PathVariable("pageId") int pageId, @PathVariable("revId") int revId) { // TODO Get the requested revision return new Revision(); } /** * Handle GET requests on /pages/{pageId}/revisions/{revId}/votes URI. All * the votes for the requested page revision are returned. * * @param pageId * int The id of the page * @param revId * int The id of the revision * @return response Vote */ @RequestMapping(value = "{pageId}/revisions/{revId}/votes", method = RequestMethod.GET, produces = "application/json") @ResponseBody public List showVotes(@PathVariable("pageId") int pageId, @PathVariable("revId") int revId) { // TODO Get votes from PageService/PageDAO return new ArrayList(); } /** * Handle POST requests on /pages/{pageId}/revisions/{revId}/votes URI. It * expects vote and userId as request parameters. * * @param pageId * int The id of the page * @param revId * int The id of the revision * @param vote * double The value of the vote * @param userId * int The id of the user who submit the request * @return response Vote */ @RequestMapping(value = "{pageId}/revisions/{revId}/votes", method = RequestMethod.POST, produces = "application/json") @ResponseBody public Vote addVote(@PathVariable("pageId") int pageId, @PathVariable("revId") int revId, @RequestParam("vote") double vote, @RequestParam("userId") int userId) { // TODO Insert the vote into the graph return new Vote(); } } diff --git a/src/main/java/org/wikitolearn/wikirating/model/Page.java b/src/main/java/org/wikitolearn/wikirating/model/Page.java index 53da3a0..8d99d34 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/Page.java +++ b/src/main/java/org/wikitolearn/wikirating/model/Page.java @@ -1,212 +1,219 @@ /** * */ package org.wikitolearn.wikirating.model; import java.util.Set; import org.neo4j.ogm.annotation.GraphId; import org.neo4j.ogm.annotation.Index; import org.neo4j.ogm.annotation.Labels; import org.neo4j.ogm.annotation.NodeEntity; import org.neo4j.ogm.annotation.Relationship; /** * @author aletundo * */ @NodeEntity( label = "Page") public class Page { @GraphId private Long graphId; private int pageid; @Index private String title; @Index private String lang; @Index(unique = true, primary = true) private String langPageId; private double pageRank; @Relationship(type = "LAST_REVISION", direction = Relationship.OUTGOING) private Revision lastRevision; @Relationship(type = "FIRST_REVISION", direction = Relationship.OUTGOING) private Revision fistRevision; @Relationship(type = "LEVEL_TWO", direction = Relationship.UNDIRECTED) private Set levelsTwo; @Relationship(type = "LEVEL_THREE", direction = Relationship.UNDIRECTED) private Set levelsThree; @Labels private Set labels; /** * */ public Page() {} /** * @param pageid * @param title * @param lang */ public Page(int pageid, String title, String lang, String langPageId) { this.pageid = pageid; this.title = title; this.lang = lang; this.langPageId = langPageId; } /** * @return the graphId */ public Long getGraphId() { return graphId; } /** * @param graphId the graphId to set */ public void setGraphId(Long graphId) { this.graphId = graphId; } /** * @return the pageid */ public int getPageid() { return pageid; } /** * @param pageid the pageid to set */ public void setPageid(int pageid) { this.pageid = pageid; } /** * @return the title */ public String getTitle() { return title; } /** * @param title the title to set */ public void setTitle(String title) { this.title = title; } /** * @return the lang */ public String getLang() { return lang; } /** * @param lang the lang to set */ public void setLang(String lang) { this.lang = lang; } /** * @return the langPageId */ public String getLangPageId() { return langPageId; } /** * @param langPageId the langPageId to set */ public void setLangPageId(String langPageId) { this.langPageId = langPageId; } /** * @return the pageRank */ public double getPageRank() { return pageRank; } /** * @param pageRank the pageRank to set */ public void setPageRank(double pageRank) { this.pageRank = pageRank; } /** * @return the lastRevision */ public Revision getLastRevision() { return lastRevision; } /** * @param lastRevision the lastRevision to set */ public void setLastRevision(Revision lastRevision) { this.lastRevision = lastRevision; } /** * @return the fistRevision */ public Revision getFistRevision() { return fistRevision; } /** * @param fistRevision the fistRevision to set */ public void setFistRevision(Revision fistRevision) { this.fistRevision = fistRevision; } /** * @return the levelsTwo */ public Set getLevelsTwo() { return levelsTwo; } /** * @param levelsTwo the levelsTwo to set */ public void setLevelsTwo(Set levelsTwo) { this.levelsTwo = levelsTwo; } /** * @return the levelsThree */ public Set getLevelsThree() { return levelsThree; } /** * @param levelsThree the levelsThree to set */ public void setLevelsThree(Set levelsThree) { this.levelsThree = levelsThree; } /** * @return the labels */ public Set getLabels() { return labels; } /** * @param labels the labels to set */ public void setLabels(Set labels) { this.labels = labels; } + + /** + * @param label the label to set + */ + public void addLabel(String label) { + this.labels.add(label); + } } \ No newline at end of file diff --git a/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java index 1736bc5..50e17ab 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java @@ -1,28 +1,37 @@ /** * */ package org.wikitolearn.wikirating.repository; +import java.util.List; + import org.springframework.data.neo4j.repository.GraphRepository; import org.wikitolearn.wikirating.model.Page; /** * @author aletundo * */ public interface PageRepository extends GraphRepository { /** * * @param title * @param lang * @return */ Page findByTitleAndLang(String title, String lang); /** * * @param langPageId * @return */ Page findByLangPageId(String langPageId); + + /** + * + * @param lang + * @return + */ + List findAllByLang(String lang); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/PageService.java b/src/main/java/org/wikitolearn/wikirating/service/PageService.java index 3a22f09..13633b5 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/PageService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/PageService.java @@ -1,111 +1,163 @@ /** * */ package org.wikitolearn.wikirating.service; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.function.Predicate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import org.wikitolearn.wikirating.model.CourseTree; import org.wikitolearn.wikirating.model.Page; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.service.mediawiki.PageMediaWikiService; /** * * @author aletundo, valsdav * */ @Service public class PageService { private static final Logger LOG = LoggerFactory.getLogger(PageService.class); @Autowired private PageMediaWikiService pageMediaWikiService; @Autowired RevisionService revisionService; @Autowired private PageRepository pageRepository; /** * This methods inserts all the pages inside the DB querying the MediaWiki API. * @param lang String * @param apiUrl String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture addAllPages( String lang, String apiUrl ){ List pages = pageMediaWikiService.getAll(apiUrl); pages.forEach(page -> { page.setLang(lang); page.setLangPageId(lang + "_" +page.getPageid()); }); pageRepository.save(pages); LOG.info("Inserted all {} pages", lang); return CompletableFuture.completedFuture(true); } /** * This method creates a new Page. It requires the firstRevision of the Page in order * to create the LAST_REVISION and FIRST_REVISION relationships. * @param pageid * @param title * @param lang * @param firstRevision * @return */ public Page addPage(int pageid, String title, String lang, Revision firstRevision){ Page page = new Page(pageid, title, lang, lang + "_" + pageid); - // Add the links from the page to the first revision + // Add the links from the page to the first and last revision page.setFistRevision(firstRevision); page.setLastRevision(firstRevision); pageRepository.save(page); return page; } /** * This method adds a new Revision to a page. It links the Page to the new revision via * LAST_REVISION link. Moreover it create the PREVIOUS_REVISION link. * @param langPageId * @param rev * @return */ public Page addRevisionToPage(String langPageId, Revision rev){ Page page = pageRepository.findByLangPageId(langPageId); - //adding PREVIOUS_REVISION relationship + // Add PREVIOUS_REVISION relationship rev.setPreviousRevision(page.getLastRevision()); page.setLastRevision(rev); - //the changes on the revision will be automatically persisted + // The changes on the revision will be automatically persisted pageRepository.save(page); return page; } /** * This method changes only the title of a given Page. * @param oldTitle * @param newTitle * @param lang * @return */ public Page movePage(String oldTitle, String newTitle, String lang){ Page page = pageRepository.findByTitleAndLang(oldTitle, lang); page.setTitle(newTitle); pageRepository.save(page); return page; } public void deletePage(String title, String lang ){ Page page = pageRepository.findByTitleAndLang(title, lang); // Delete the revisions of the page revisionService.deleteRevisionsOfPage(page.getLangPageId()); // Delete finally the page itself pageRepository.delete(page); //TODO throw specific exceptions } + + /** + * Apply the course structure using labels and relationships to the graph + * @param apiUrl + * @param lang + */ + public void applyCourseStructure(String apiUrl, String lang){ + List pages = pageRepository.findAllByLang(lang); + + // Remove all pages that are not course root pages + Predicate pagePredicate = page -> page.getTitle().contains("/"); + pages.removeIf(pagePredicate); + + for(Page p : pages){ + // Get course tree and prepare relationship sets + CourseTree tree = pageMediaWikiService.getCourseTree(apiUrl, p.getTitle()); + Set levelsTwo = new HashSet<>(); + Set levelsThree = new HashSet<>(); + + int index = 0; + for(String levelTwo : tree.getLevelsTwo()){ + // Add CourseLEvelTwo label and add levels two to the set to be saved + String levelTwoTitle = (tree.getRoot() + "/" + levelTwo).trim(); + Page levelTwoPage = pageRepository.findByTitleAndLang(levelTwoTitle, lang); + levelTwoPage.addLabel("CourseLevelTwo"); + levelsTwo.add(levelTwoPage); + + // Add CourseLevelThree labels and add levels three to the set to be saved + for(String levelThree : tree.getLevelsTree().get(index)){ + String levelThreeTitle = (levelTwoTitle + "/" + levelThree).trim(); + Page levelTwoThree = pageRepository.findByTitleAndLang(levelThreeTitle, lang); + levelTwoThree.addLabel("CourseLevelThree"); + levelsThree.add(levelTwoPage); + } + // Set LEVEL_THREE relationships + levelTwoPage.setLevelsThree(levelsThree); + index++; + } + // Set LEVEL_TWO relationships and CourseRoot label + p.addLabel("CourseRoot"); + p.setLevelsTwo(levelsTwo); + + // Save all the course structure + pageRepository.save(levelsThree); + pageRepository.save(levelsTwo); + pageRepository.save(p); + } + } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/PageMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/PageMediaWikiService.java index b6be31b..5d5d8de 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/PageMediaWikiService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/PageMediaWikiService.java @@ -1,104 +1,107 @@ /** * */ package org.wikitolearn.wikirating.service.mediawiki; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.stereotype.Service; import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikitolearn.wikirating.exception.GenericException; import org.wikitolearn.wikirating.model.CourseTree; import org.wikitolearn.wikirating.model.Page; import com.fasterxml.jackson.core.type.TypeReference; /** * * @author aletundo, valsdav * */ @Service public class PageMediaWikiService extends MediaWikiService{ /** * Get all the pages from a specified namespace of MediaWiki instance through its API. * @param apiUrl String The MediaWiki API url * @return pages List A list that contains all the fetched pages */ @Override public List getAll(String apiUrl){ ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); - Map parameters = mediaWikiApiUtils.getListAllPagesParams(namespace); + Map parameters = mediaWikiApiUtils.getListAllPagesParams("2800"); InputStream response; boolean morePages = true; JSONArray pagesJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List pages = new ArrayList<>(); try { while(morePages){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("allpages")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("apcontinue"); parameters.put("apfrom", continueFrom); }else{ morePages = false; pagesJson = concatArrays(toBeConcat); } } pages = mapper.readValue(pagesJson.toString(), new TypeReference>(){}); return pages; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return pages; } /** - * + * Get the course tree structure through the MediaWiki API * @param apiUrl the MediaWiki API url * @param pageTitle the title of the root course page - * @return + * @return the course tree */ - public void getCourseTree(String apiUrl, String pageTitle) { + public CourseTree getCourseTree(String apiUrl, String pageTitle) { ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); Map parameters = mediaWikiApiUtils.getCourseTreeParams(pageTitle); InputStream response; response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); try { JSONObject jsonTree = responseJson.getJSONObject("coursetree").getJSONObject("response"); List> levelsThree = new ArrayList<>(); JSONArray levelsThreeJson = jsonTree.getJSONArray("levelsThree"); levelsThree = mapper.readValue(levelsThreeJson.toString(), new TypeReference>>(){}); // Build course tree manually cause difficulties with serialization of nested JSON arrays CourseTree courseTree = new CourseTree(); courseTree.setRoot(jsonTree.getString("root")); courseTree.setLevelsTwo( mapper.readValue( jsonTree.getJSONArray("levelsTwo").toString(), new TypeReference>(){} ) ); courseTree.setLevelsTree(levelsThree); - LOG.info(courseTree.toString()); + LOG.info("Got course tree for page {}: {}", pageTitle, courseTree.toString()); + return courseTree; } catch (JSONException | IOException e) { LOG.error("An error occurred: {}", e.getMessage()); + throw new GenericException(e.getMessage()); } } }