diff --git a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java index 7b4e100..959aa46 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java @@ -1,24 +1,35 @@ /** * */ package org.wikitolearn.wikirating.repository; import java.util.Set; +import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.repository.GraphRepository; import org.wikitolearn.wikirating.model.Revision; /** * @author aletundo * */ public interface RevisionRepository extends GraphRepository { /** * * @param langRevId * @return */ Revision findByLangRevId(String langRevId); Set findByUserid(int userId); + + /** + * This query returns all the Revisions of a Page. + * The direction -> of the link is important to transverse + * only the chain of Revisions of the page without reaching other nodes. + * @param langPageId + * @return + */ + @Query("MATCH (m:Metadata)-[*]->(p:Process {type:{0}}) RETURN p LIMIT 1") + Set findAllRevisionOfPage(String langPageId); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/PageService.java b/src/main/java/org/wikitolearn/wikirating/service/PageService.java index 704470b..3a22f09 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/PageService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/PageService.java @@ -1,103 +1,111 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.List; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.model.Page; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.service.mediawiki.PageMediaWikiService; /** * * @author aletundo, valsdav * */ @Service public class PageService { private static final Logger LOG = LoggerFactory.getLogger(PageService.class); @Autowired private PageMediaWikiService pageMediaWikiService; @Autowired RevisionService revisionService; @Autowired private PageRepository pageRepository; /** * This methods inserts all the pages inside the DB querying the MediaWiki API. * @param lang String * @param apiUrl String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture addAllPages( String lang, String apiUrl ){ List pages = pageMediaWikiService.getAll(apiUrl); pages.forEach(page -> { page.setLang(lang); page.setLangPageId(lang + "_" +page.getPageid()); }); pageRepository.save(pages); LOG.info("Inserted all {} pages", lang); return CompletableFuture.completedFuture(true); } /** * This method creates a new Page. It requires the firstRevision of the Page in order * to create the LAST_REVISION and FIRST_REVISION relationships. * @param pageid * @param title * @param lang * @param firstRevision * @return */ public Page addPage(int pageid, String title, String lang, Revision firstRevision){ Page page = new Page(pageid, title, lang, lang + "_" + pageid); - + // Add the links from the page to the first revision page.setFistRevision(firstRevision); page.setLastRevision(firstRevision); pageRepository.save(page); return page; } /** * This method adds a new Revision to a page. It links the Page to the new revision via * LAST_REVISION link. Moreover it create the PREVIOUS_REVISION link. * @param langPageId * @param rev * @return */ public Page addRevisionToPage(String langPageId, Revision rev){ Page page = pageRepository.findByLangPageId(langPageId); //adding PREVIOUS_REVISION relationship rev.setPreviousRevision(page.getLastRevision()); page.setLastRevision(rev); //the changes on the revision will be automatically persisted pageRepository.save(page); return page; } /** * This method changes only the title of a given Page. * @param oldTitle * @param newTitle * @param lang * @return */ public Page movePage(String oldTitle, String newTitle, String lang){ Page page = pageRepository.findByTitleAndLang(oldTitle, lang); page.setTitle(newTitle); pageRepository.save(page); return page; } + public void deletePage(String title, String lang ){ + Page page = pageRepository.findByTitleAndLang(title, lang); + // Delete the revisions of the page + revisionService.deleteRevisionsOfPage(page.getLangPageId()); + // Delete finally the page itself + pageRepository.delete(page); + //TODO throw specific exceptions + } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index a5acf5e..269fa23 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,88 +1,94 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.List; import java.util.ListIterator; +import java.util.Set; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.model.Page; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private PageRepository pageRepository; /** * This method inserts all the revisions for every page, creating the * connections between them and between the users that have written them. * * @param lang * String * @param apiUrl * String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture addAllRevisions(String lang, String apiUrl) { Iterable pages = pageRepository.findAll(); pages.forEach(page -> { List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageid()); // Set the first and the last revisions for the current page page.setFistRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevid()); rev.setLang(lang); if (it.previousIndex() != 0){ rev.setPreviousRevision(revisions.get(it.previousIndex()-1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); pageRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); }); return CompletableFuture.completedFuture(true); } /** * This method adds a new Revision to the DB. * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang,userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } + public void deleteRevisionsOfPage(String langPageId){ + Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); + revisionRepository.delete(revisions); + //TODO throw exceptions + } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java index b516c74..9527682 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java @@ -1,128 +1,130 @@ package org.wikitolearn.wikirating.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.model.Process; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.model.UpdateInfo; import org.wikitolearn.wikirating.model.User; import org.wikitolearn.wikirating.service.mediawiki.UpdateMediaWikiService; import org.wikitolearn.wikirating.util.enums.ProcessStatus; import org.wikitolearn.wikirating.util.enums.ProcessType; import java.util.ArrayList; import java.util.Date; import java.util.List; /** * Created by valsdav on 29/03/17. */ @Service public class UpdateService { private static final Logger LOG = LoggerFactory.getLogger(UpdateService.class); @Autowired private UserService userService; @Autowired private PageService pageService; @Autowired private RevisionService revisionService; @Autowired private ProcessService processService; @Autowired private UpdateMediaWikiService updateMediaWikiService; @Value("#{'${mediawiki.langs}'.split(',')}") private List langs; @Value("${mediawiki.protocol}") private String protocol; @Value("${mediawiki.api.url}") private String apiUrl; @Value("${mediawiki.namespace}") private String namespace; /** * * @return */ public boolean updateGraph() { // Get start timestamp of the latest FETCH Process before opening a new process Date startTimestampLatestFetch = processService.getLastProcessStartDateByType(ProcessType.FETCH); // Create a new FETCH process Process currentFetchProcess = processService.createNewProcess(ProcessType.FETCH); LOG.info("Created new fetch process {}", currentFetchProcess.toString()); Date startTimestampCurrentFetch = currentFetchProcess.getStartOfProcess(); updatePagesAndRevisions(startTimestampLatestFetch, startTimestampCurrentFetch); updateUsers(startTimestampLatestFetch, startTimestampCurrentFetch); // Save the result of the process, closing the current one processService.closeCurrentProcess(ProcessStatus.DONE); return true; } /** * @param start * @param end */ private void updateUsers(Date start, Date end) { String url = protocol + langs.get(0) + "." + apiUrl; List usersUpdateInfo = updateMediaWikiService.getNewUsers(url, start, end); List newUsers = new ArrayList<>(); // Build a list with new users to be added to the graph for(UpdateInfo updateInfo : usersUpdateInfo){ User user = new User(); user.setUserId(updateInfo.getUserid()); user.setUsername(updateInfo.getUser()); newUsers.add(user); } userService.addUsers(newUsers); } /** * * @param start * @param end */ private void updatePagesAndRevisions(Date start, Date end) { // First of all, get the RecentChangeEvents from MediaWiki API for (String lang : langs) { String url = protocol + lang + "." + apiUrl; // Fetching pages updates List updates = updateMediaWikiService.getPagesUpdateInfo(url, namespace, start, end); updates.forEach(update -> { switch (update.getType()) { case NEW: // Create the new revision Revision newRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Then create a new Page and link it with the revision pageService.addPage(update.getPageid(), update.getTitle(), lang, newRev); break; case EDIT: // Create a new revision Revision updateRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Then add it to the page pageService.addRevisionToPage(lang + "_" + update.getPageid(), updateRev); break; case MOVE: // Move the page to the new title pageService.movePage(update.getTitle(), update.getNewTitle(), lang); break; case DELETE: + // Delete the page and all its revisions + pageService.deletePage(update.getTitle(), lang); break; default: break; } }); } } }