diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index ffac117..c15c602 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,122 +1,122 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.List; import java.util.ListIterator; import java.util.Set; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.model.Page; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private PageRepository pageRepository; /** * Initialize the revisions for the first time querying the MediaWiki API. * This method adds the revisions and sets the FIRST_REVISION, * LAST_REVISION and PREVIOUS_REVISION relationships. * @param lang the domain language * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initRevisions(String lang, String apiUrl) { Iterable pages = pageRepository.findAll(); pages.forEach(page -> { List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageid()); // Set the first and the last revisions for the current page page.setFistRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevid()); rev.setLang(lang); if (it.previousIndex() != 0){ rev.setPreviousRevision(revisions.get(it.previousIndex()-1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); pageRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); }); return CompletableFuture.completedFuture(true); } /** * Add a new Revision to the graph * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang,userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } /** * Delete a revision given its langPageId * @param langPageId the langPageId of the revision */ public void deleteRevisionsOfPage(String langPageId){ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); revisionRepository.delete(revisions); //TODO throw exceptions } /** * Get the requested revision - * @param langRevId - * @return + * @param langRevId the langRevId of the revision + * @return the revision * @throws RevisionNotFoundException */ public Revision getRevision(String langRevId) throws RevisionNotFoundException{ Revision revision = revisionRepository.findByLangRevId(langRevId); if(revision == null){ LOG.error("Revision {} not found", langRevId); throw new RevisionNotFoundException(); } return revision; } /** * Update the given revision * @param revision * @return the updated revision */ public Revision updateRevision(Revision revision){ revisionRepository.save(revision); return revision; } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java index 96b6701..daa261a 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java @@ -1,172 +1,173 @@ package org.wikitolearn.wikirating.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.exception.TemporaryVoteValidationException; import org.wikitolearn.wikirating.exception.UpdateGraphException; import org.wikitolearn.wikirating.exception.UserNotFoundException; import org.wikitolearn.wikirating.model.Process; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.model.TemporaryVote; import org.wikitolearn.wikirating.model.UpdateInfo; import org.wikitolearn.wikirating.model.User; import org.wikitolearn.wikirating.model.Vote; import org.wikitolearn.wikirating.repository.TemporaryVoteRepository; import org.wikitolearn.wikirating.service.mediawiki.UpdateMediaWikiService; import org.wikitolearn.wikirating.util.enums.ProcessStatus; import org.wikitolearn.wikirating.util.enums.ProcessType; import java.util.ArrayList; import java.util.Date; import java.util.List; /** * Created by valsdav on 29/03/17. */ @Service public class UpdateService { private static final Logger LOG = LoggerFactory.getLogger(UpdateService.class); @Autowired private UserService userService; @Autowired private PageService pageService; @Autowired private RevisionService revisionService; @Autowired private ProcessService processService; @Autowired private UpdateMediaWikiService updateMediaWikiService; @Autowired private TemporaryVoteRepository temporaryVoteRepository; @Value("#{'${mediawiki.langs}'.split(',')}") private List langs; @Value("${mediawiki.protocol}") private String protocol; @Value("${mediawiki.api.url}") private String apiUrl; @Value("${mediawiki.namespace}") private String namespace; /** * Entry point for the scheduled graph updated * @return true if the update succeed */ @Scheduled(cron = "${maintenance.update.cron}") public boolean updateGraph() { // Get start timestamp of the latest FETCH Process before opening a new process Date startTimestampLatestFetch = processService.getLastProcessStartDateByType(ProcessType.FETCH); // Create a new FETCH process Process currentFetchProcess = processService.createNewProcess(ProcessType.FETCH); LOG.info("Created new fetch process {}", currentFetchProcess.toString()); Date startTimestampCurrentFetch = currentFetchProcess.getStartOfProcess(); try{ updateUsers(startTimestampLatestFetch, startTimestampCurrentFetch); updatePagesAndRevisions(startTimestampLatestFetch, startTimestampCurrentFetch); validateTemporaryVotes(startTimestampCurrentFetch); }catch(TemporaryVoteValidationException e){ LOG.error("An error occurred during a scheduled graph update procedure"); throw new UpdateGraphException(); } // Save the result of the process, closing the current one processService.closeCurrentProcess(ProcessStatus.DONE); return true; } /** * Update the users querying the MediaWiki API * @param start * @param end */ private void updateUsers(Date start, Date end) { String url = protocol + langs.get(0) + "." + apiUrl; List usersUpdateInfo = updateMediaWikiService.getNewUsers(url, start, end); List newUsers = new ArrayList<>(); // Build a list with new users to be added to the graph for(UpdateInfo updateInfo : usersUpdateInfo){ User user = new User(); user.setUserId(updateInfo.getUserid()); user.setUsername(updateInfo.getUser()); newUsers.add(user); } userService.addUsers(newUsers); } /** * Update the pages and the revisions querying the MediaWiki API * @param start * @param end */ private void updatePagesAndRevisions(Date start, Date end) { // First of all, get the RecentChangeEvents from MediaWiki API for (String lang : langs) { String url = protocol + lang + "." + apiUrl; // Fetching pages updates List updates = updateMediaWikiService.getPagesUpdateInfo(url, namespace, start, end); for(UpdateInfo update : updates){ switch (update.getType()) { case NEW: // Create the new revision Revision newRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Then create a new Page and link it with the revision pageService.addPage(update.getPageid(), update.getTitle(), lang, newRev); userService.setAuthorship(newRev); break; case EDIT: // Create a new revision Revision updateRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Then add it to the page pageService.addRevisionToPage(lang + "_" + update.getPageid(), updateRev); userService.setAuthorship(updateRev); break; case MOVE: // Move the page to the new title pageService.movePage(update.getTitle(), update.getNewTitle(), lang); break; case DELETE: // Delete the page and all its revisions pageService.deletePage(update.getTitle(), lang); break; default: break; } } } } /** * Validate temporary votes added before the given timestamp * @param timestamp the timestamp used for comparison + * @throws TemporaryVoteValidationException */ private void validateTemporaryVotes(Date timestamp) throws TemporaryVoteValidationException{ List temporaryVotes = temporaryVoteRepository.findByTimestamp(timestamp); for(TemporaryVote temporaryVote: temporaryVotes){ try{ User user = userService.getUser(temporaryVote.getUserid()); Revision revision = revisionService.getRevision(temporaryVote.getLangRevId()); Vote vote = new Vote(temporaryVote.getValue(), temporaryVote.getReliability(), temporaryVote.getTimestamp()); vote.setRevision(revision); vote.setUser(user); revision.addVote(vote); revisionService.updateRevision(revision); }catch(UserNotFoundException | RevisionNotFoundException e){ LOG.error("An error occurred during temporary vote validation: {}", temporaryVote); throw new TemporaryVoteValidationException(e.getMessage()); } } } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/UserService.java b/src/main/java/org/wikitolearn/wikirating/service/UserService.java index 2a696cc..a48dfec 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/UserService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/UserService.java @@ -1,120 +1,120 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.UserNotFoundException; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.model.User; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.repository.UserRepository; import org.wikitolearn.wikirating.service.mediawiki.UserMediaWikiService; /** * * @author aletundo, valsdav * */ @Service public class UserService { private static final Logger LOG = LoggerFactory.getLogger(UserService.class); @Autowired private UserMediaWikiService userMediaWikiService; @Autowired private UserRepository userRepository; @Autowired private RevisionRepository revisionRepository; /** * Initialize the graph for the first time querying the MediaWiki API * to get the all the users and then insert them. * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initUsers(String apiUrl){ List users = userMediaWikiService.getAll(apiUrl); addUsers(users); return CompletableFuture.completedFuture(true); } /** * Initialize relationships between users and their created revisions for the first time * @return CompletableFuture */ @Async public CompletableFuture initAuthorship(){ Iterable users = userRepository.findAll(); for(User user : users){ Set revisions = revisionRepository.findByUserid(user.getUserid()); user.setRevisionsAuthored(revisions); userRepository.save(user); LOG.info("Set revisions authorship for user {}", user.getUserid()); } // Get revisions with userid = 0 (anonymous authors) Set anonRevisions = revisionRepository.findByUserid(0); User anonymousUser = new User("AnonymousUser", 0, 0.0, 0.0, 0.0); anonymousUser.setRevisionsAuthored(anonRevisions); userRepository.save(anonymousUser); LOG.info("Set revisions authorship for anonymous revisions"); return CompletableFuture.completedFuture(true); } /** * Insert users into the graph * @param users the list of users to be inserted * @return the list of inserted users */ public List addUsers(List users){ userRepository.save(users); LOG.info("Inserted users: {}", users); return users; //TODO handle exceptions } /** * Set the authorship for a revision * @param revision the revision */ public void setAuthorship(Revision revision){ User user = userRepository.findByUserId(revision.getUserid()); user.setRevisionAuthored(revision); } /** * Set the authorship for a list of revisions * @param revisions the list of revisions */ public void setAuthorship(List revisions){ for(Revision revision : revisions){ setAuthorship(revision); } } /** * Get the requested user. * @param userId the user id - * @return the requested user if is found + * @return the requested user * @throws UserNotFoundException */ public User getUser(int userId) throws UserNotFoundException{ User user = userRepository.findByUserId(userId); if(user == null){ LOG.error("User {} not found", userId); throw new UserNotFoundException(); } return user; } }