diff --git a/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java b/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java index df5de15..81a1c02 100644 --- a/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java +++ b/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java @@ -1,130 +1,139 @@ /** * */ package org.wikitolearn.wikirating.controller; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStream; import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.util.DefaultPropertiesPersister; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; import org.wikitolearn.wikirating.exception.UpdateGraphException; import org.wikitolearn.wikirating.model.api.ApiResponse; import org.wikitolearn.wikirating.model.api.ApiResponseError; import org.wikitolearn.wikirating.model.api.ApiResponseFail; import org.wikitolearn.wikirating.model.api.ApiResponseSuccess; import org.wikitolearn.wikirating.service.MaintenanceService; /** * * @author aletundo * */ @RestController public class MaintenanceController { private static final Logger LOG = LoggerFactory.getLogger(MaintenanceController.class); @Autowired private MaintenanceService maintenanceService; /** * Secured endpoint to enable or disable read only mode. When read only mode * is enabled only GET requests are allowed. To change this behavior @see * org.wikitolearn.wikirating.wikirating.filter.MaintenanceFilter. * * @param active * String requested parameter that toggle the re. Binary values * are accepted. * @return a JSON object with the response */ @RequestMapping(value = "${maintenance.readonlymode.uri}", method = RequestMethod.POST, produces = "application/json") @ResponseBody @ResponseStatus(HttpStatus.ACCEPTED) public ApiResponse toggleReadOnlyMode(@RequestParam(value = "active") String active) { int mode = Integer.parseInt(active); ApiResponse body; if (mode == 0) { // Delete maintenance lock file if it exists File f = new File("maintenance.lock"); if (f.exists()) { f.delete(); LOG.debug("Deleted maintenance lock file."); } LOG.info("Application is live now."); body = new ApiResponseSuccess("Application is live now", Instant.now().toEpochMilli()); } else if (mode == 1) { try { // Create maintenance lock file with a maintenance.active // property set to true Properties props = new Properties(); props.setProperty("maintenance.active", "true"); File lockFile = new File("maintenance.lock"); OutputStream out = new FileOutputStream(lockFile); DefaultPropertiesPersister p = new DefaultPropertiesPersister(); p.store(props, out, "Maintenance mode lock file"); LOG.debug("Created maintenance lock file."); LOG.info("Application is in maintenance mode now."); body = new ApiResponseSuccess("Application is in maintenance mode now", Instant.now().toEpochMilli()); } catch (Exception e) { LOG.error("Something went wrong. {}", e.getMessage()); Map data = new HashMap<>(); data.put("error", HttpStatus.NOT_FOUND.name()); data.put("exception", e.getClass().getCanonicalName()); data.put("stacktrace", e.getStackTrace()); return new ApiResponseError(data, e.getMessage(), HttpStatus.NOT_FOUND.value(), Instant.now().toEpochMilli()); } } else { // The active parameter value is not supported body = new ApiResponseFail("Parameter value not supported", Instant.now().toEpochMilli()); } return body; } /** * Secured endpoint that handles initialization request for the given * language * * @return true if the initialization is completed without errors, false * otherwise */ @RequestMapping(value = "${maintenance.init.uri}", method = RequestMethod.POST, produces = "application/json") public boolean initialize() { return maintenanceService.initializeGraph(); } @RequestMapping(value = "${maintenance.fetch.uri}", method = RequestMethod.POST, produces = "application/json") public boolean fetch() { try { return maintenanceService.updateGraph(); }catch (UpdateGraphException e){ return false; } } + @RequestMapping(value = "${maintenance.compute.uri}", method = RequestMethod.POST, produces = "application/json") + public boolean compute() { + try { + return maintenanceService.computePageRanking(); + }catch (UpdateGraphException e){ + return false; + } + } + /** * */ @RequestMapping(value = "${maintenance.wipe.uri}", method = RequestMethod.DELETE, produces = "application/json") public void wipeDatabase() { // TODO } } diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java b/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java index fca86f6..ebdffb6 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java @@ -1,360 +1,374 @@ package org.wikitolearn.wikirating.model.graph; import org.neo4j.ogm.annotation.GraphId; import org.neo4j.ogm.annotation.Index; import org.neo4j.ogm.annotation.NodeEntity; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import org.neo4j.ogm.annotation.Relationship; import org.neo4j.ogm.annotation.typeconversion.DateLong; //import org.springframework.data.annotation.Transient; import java.util.Date; import java.util.Set; /** * This class handles the data of the Revision of a page. * Created by valsdav on 14/03/17. */ @JsonIgnoreProperties(ignoreUnknown = true) @NodeEntity( label = "Revision" ) public class Revision { @GraphId @JsonIgnore private Long graphId; @Index(unique = true, primary=true) @JsonProperty(access = Access.WRITE_ONLY) private String langRevId; @JsonProperty("revid") private int revId; @JsonProperty(access = Access.WRITE_ONLY) private String lang; @JsonProperty(value = "userid", access = Access.WRITE_ONLY) private int userId; @JsonProperty(value = "parentid", access = Access.WRITE_ONLY) private int parentId; @DateLong @JsonProperty(access = Access.WRITE_ONLY) private Date timestamp; @JsonProperty(value = "size", access = Access.WRITE_ONLY) private long length; @JsonProperty(access = Access.WRITE_ONLY) private double changeCoefficient; private double currentMeanVote; private double currentVotesReliability; private double totalMeanVote; private double totalVotesReliability; private double normalizedNumberOfVotes; private double currentScore; private double totalScore; @Relationship(type="PREVIOUS_REVISION", direction = Relationship.OUTGOING) private Revision previousRevision; + //@Relationship(type="PREVISOU_REVISION", direction = Relationship.INCOMING) + //private Revision nextRevision; @Relationship(type="VOTE", direction = Relationship.INCOMING) private Set votes; //This relationship represents the connection between the revision //and its creator (user). It saves his reliability at the moment of the author. @Relationship(type="AUTHOR", direction = Relationship.INCOMING) private Author author; /** * */ public Revision() {} /** * @param revId * @param userId * @param parentId * @param length * @param lang * @param timestamp */ public Revision(int revId, String lang, int userId, int parentId, long length, Date timestamp) { this.revId = revId; this.langRevId = lang +"_"+revId; this.userId = userId; this.parentId = parentId; this.length = length; this.lang = lang; this.timestamp = timestamp; this.changeCoefficient = 0.0; } /** * @return the revId */ public int getRevId() { return revId; } /** * @param revId the revId to set */ public void setRevId(int revId) { this.revId = revId; } /** * @return the userId */ public int getUserId() { return userId; } /** * @param userId the userId to set */ public void setUserId(int userId) { this.userId = userId; } /** * @return the parentId */ public int getParentId() { return parentId; } /** * @param parentId the parentId to set */ public void setParentId(int parentId) { this.parentId = parentId; } /** * @return the length */ public long getLength() { return length; } /** * @param length the length to set */ public void setLength(long length) { this.length = length; } /** * @return the changeCoefficient */ public double getChangeCoefficient() { return changeCoefficient; } /** * @param changeCoefficient the changeCoefficient to set */ public void setChangeCoefficient(double changeCoefficient) { this.changeCoefficient = changeCoefficient; } /** * @return the currentMeanVote */ public double getCurrentMeanVote() { return currentMeanVote; } /** * @param currentMeanVote the currentMeanVote to set */ public void setCurrentMeanVote(double currentMeanVote) { this.currentMeanVote = currentMeanVote; } /** * @return the currentVotesReliability */ public double getCurrentVotesReliability() { return currentVotesReliability; } /** * @param currentVotesReliability the currentVotesReliability to set */ public void setCurrentVotesReliability(double currentVotesReliability) { this.currentVotesReliability = currentVotesReliability; } /** * @return the totalMeanVote */ public double getTotalMeanVote() { return totalMeanVote; } /** * @param totalMeanVote the totalMeanVote to set */ public void setTotalMeanVote(double totalMeanVote) { this.totalMeanVote = totalMeanVote; } /** * @return the totalVotesReliability */ public double getTotalVotesReliability() { return totalVotesReliability; } /** * @param totalVotesReliability the totalVotesReliability to set */ public void setTotalVotesReliability(double totalVotesReliability) { this.totalVotesReliability = totalVotesReliability; } public double getCurrentScore() { return currentScore; } public void setCurrentScore(double currentScore) { this.currentScore = currentScore; } public double getTotalScore() { return totalScore; } public void setTotalScore(double totalScore) { this.totalScore = totalScore; } /** * @return the lang */ public String getLang() { return lang; } /** * @param lang the lang to set */ public void setLang(String lang) { this.lang = lang; } /** * @return the langRevId */ public String getLangRevId() { return langRevId; } /** * @param langRevId the langRevId to set */ public void setLangRevId(String langRevId) { this.langRevId = langRevId; } /** * @return the graphId */ public Long getGraphId() { return graphId; } /** * @param graphId the graphId to set */ public void setGraphId(Long graphId) { this.graphId = graphId; } /** * @return the timestamp */ public Date getTimestamp() { return timestamp; } /** * @param timestamp the timestamp to set */ public void setTimestamp(Date timestamp) { this.timestamp = timestamp; } /** * @return the previousRevision */ public Revision getPreviousRevision() { return previousRevision; } /** * @param previousRevision the previousRevision to set */ public void setPreviousRevision(Revision previousRevision) { this.previousRevision = previousRevision; } + /* + public Revision getNextRevision() { + return nextRevision; + } + + public void setNextRevision(Revision nextRevision) { + this.nextRevision = nextRevision; + } +*/ + /** * @return the votes */ public Set getVotes() { return votes; } /** * @param votes the votes to set */ public void setVotes(Set votes) { this.votes = votes; } public Author getAuthor() { return author; } public void setAuthor(Author author) { this.author = author; } public double getNormalizedNumberOfVotes() { return normalizedNumberOfVotes; } public void setNormalizedNumberOfVotes(double normalizedNumberOfVotes) { this.normalizedNumberOfVotes = normalizedNumberOfVotes; } - public int getCurrentNumberOfVotes() { + public int getCurrentNumberOfVotes() { if (votes != null) { return this.votes.size(); }else{ return 0; } } /** * * @param vote the vote to set */ public void addVote(Vote vote){ this.votes.add(vote); } /* (non-Javadoc) * @see java.lang.Object#toString() */ @Override public String toString() { return "Revision{" + "langRevId=" + langRevId + ", userId=" + userId + ", timestamp=" + timestamp + ", length=" + length + ", author=" + author + '}'; } + + } \ No newline at end of file diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/queryresult/RevisionResult.java b/src/main/java/org/wikitolearn/wikirating/model/graph/queryresult/RevisionResult.java new file mode 100644 index 0000000..5a75e64 --- /dev/null +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/queryresult/RevisionResult.java @@ -0,0 +1,12 @@ +package org.wikitolearn.wikirating.model.graph.queryresult; + +import org.springframework.data.neo4j.annotation.QueryResult; +import org.wikitolearn.wikirating.model.graph.Revision; + +/** + * Created by valsdav on 09/05/17. + */ +@QueryResult +public class RevisionResult { + public Revision revision; +} diff --git a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java index 5a8946f..c90d569 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java @@ -1,65 +1,92 @@ /** * */ package org.wikitolearn.wikirating.repository; +import java.util.List; import java.util.Set; import org.springframework.data.neo4j.annotation.Query; +import org.springframework.data.neo4j.annotation.QueryResult; import org.springframework.data.neo4j.repository.GraphRepository; +import org.springframework.data.repository.query.Param; import org.wikitolearn.wikirating.model.graph.Revision; +import org.wikitolearn.wikirating.model.graph.queryresult.RevisionResult; /** * @author aletundo * */ public interface RevisionRepository extends GraphRepository { /** * * @param langRevId * @return */ Revision findByLangRevId(String langRevId); + @Query("MATCH (r:Revision {langRevId:{0}}) WITH r MATCH p=(r)-[s*0..1]-() RETURN r as revision, nodes(p), relationships(p)") + RevisionResult getRev(String id); /** * * @param lang * @return */ Set findByLang(String lang); /** * * @param userId * @return */ Set findByUserId(int userId); /** * This query returns all the Revisions of a Page (CourseLevelThree). * The direction -> of the link is important to traverse * only the chain of Revisions of the page without reaching other nodes. * @param langPageId * @return */ @Query("MATCH (p:CourseLevelThree {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r") Set findAllRevisionOfPage(String langPageId); /** * This query returns all the Revisions of a Page ordere from the oldest * to the newest. * @param langPageId * @return */ @Query("MATCH (p:CourseLevelThree {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r ORDER BY r.revId") - Set findAllRevisionOfPageOrdered(String langPageId); + List findAllRevisionOfPageOrdered(String langPageId); + + /** + * This query returns all the Revisions after the LAST_VALIDATED_REVISION. + * The revisions are returned ordered by revId from the oldest to the newest. + * @param langPageId + * @return + */ + @Query("MATCH (page:Page {langPageId:{0}})-[:LAST_VALIDATED_REVISION]->(rev:Revision) " + + "WITH rev MATCH (rev)<-[:PREVIOUS_REVISION*]-(rev:Revision) " + + "WITH pr MATCH path=(pr)-[r*0..1]-() " + + "RETURN pr as revision , nodes(path), relationships(path) ORDER BY pr.revId") + List findAllRevisionNotValidated(String langPageId); + + /** + * + */ + @Query("MATCH (Page {langPageId:{id}})-[:LAST_VALIDATED_REVISION]->(rev:Revision) " + + "WITH rev MATCH (rev)-[a:AUTHOR]-(s:User) " + + "RETURN rev as revision, a, s as author ") + RevisionResult findLastValidatedRevision(@Param("id") String langPageId); /** * This query return the previous revision of a Revision identified * by langRevId. * @param langRevId langRevId of the requested revision * @return the previous revision */ @Query("MATCH (r:Revision {langRevId:{0}})-[:PREVIOUS_REVISION]->(a:Revision) RETURN a") Revision findPreviousRevision(String langRevId); + } diff --git a/src/main/java/org/wikitolearn/wikirating/service/ComputeService.java b/src/main/java/org/wikitolearn/wikirating/service/ComputeService.java index 205a077..55ab61d 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/ComputeService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/ComputeService.java @@ -1,104 +1,111 @@ package org.wikitolearn.wikirating.service; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import org.wikitolearn.wikirating.model.graph.Revision; import org.wikitolearn.wikirating.model.graph.Vote; +import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; /** * Created by valsdav on 07/05/17. */ @Service public class ComputeService { @Autowired PageService pageService; @Autowired RevisionService revisionService; @Autowired UserService userService; @Autowired VoteService voteService; - @Async - public CompletableFuture calculateRevisionsPoints(String langPageId){ + //@Async + public CompletableFuture computeRevisionsRating(String langPageId){ // Get all the revision ordered from the oldest - Set revisions = revisionService.getRevisionsOfPageOrdered(langPageId); + List revisions = revisionService.getRevisionsFromLastValidated(langPageId); // The order of the revision ensures that we have always the values - // for the previous revision. + // for the previous revision. for (Revision revision : revisions){ calculateVotesMean(revision); calculateTotalVotesMean(revision); calculateSigmaAndScore(revision); calculateTotalSigmaAndScore(revision); } + revisionService.updateRevisions(revisions); return CompletableFuture.completedFuture(true); } private void calculateVotesMean(Revision revision){ double total = 0; double weights = 0; - for (Vote vote : revision.getVotes()){ - double reliabilitySquared = Math.pow(vote.getReliability(),2); - total += reliabilitySquared * vote.getValue(); - weights += reliabilitySquared; + if (revision.getCurrentNumberOfVotes() > 0){ + for (Vote vote : revision.getVotes()){ + double reliabilitySquared = Math.pow(vote.getReliability(),2); + total += reliabilitySquared * vote.getValue(); + weights += reliabilitySquared; + } + double currentMean = total/weights; + revision.setCurrentMeanVote(currentMean); + } else { + revision.setCurrentMeanVote(0.0); } - double currentMean = total/weights; - revision.setCurrentMeanVote(currentMean); } private void calculateTotalVotesMean(Revision revision){ Revision previousRevision = revision.getPreviousRevision(); if (previousRevision == null){ // First revision of the page revision.setTotalMeanVote(revision.getCurrentMeanVote()); }else { double changeCoefficient = revision.getChangeCoefficient(); // Calculate total mean double previousNVotes = previousRevision.getNormalizedNumberOfVotes(); int currentNVotes = revision.getCurrentNumberOfVotes(); double total = (previousNVotes * changeCoefficient) * previousRevision.getTotalMeanVote(); total += currentNVotes * revision.getCurrentMeanVote(); double weights = previousNVotes * changeCoefficient + currentNVotes; - revision.setTotalMeanVote(total/weights); + revision.setTotalMeanVote(total/weights); // Calculate normalized number of votes revision.setNormalizedNumberOfVotes(previousNVotes * changeCoefficient + currentNVotes); } } private void calculateSigmaAndScore(Revision revision){ double totalMean = revision.getTotalMeanVote(); double sigmaTotal = 0; double score = 0; for (Vote vote : revision.getVotes()){ double deltaVote = totalMean - vote.getValue(); double sigma = Math.pow(vote.getReliability(),2)* (1 - Math.pow(deltaVote,2)); sigmaTotal += sigma; score += sigma * Math.pow(vote.getValue(), 2); } revision.setCurrentVotesReliability(sigmaTotal); revision.setCurrentScore(score); } private void calculateTotalSigmaAndScore(Revision revision){ Revision previousRevision = revision.getPreviousRevision(); if (previousRevision == null){ // First revision of the page revision.setTotalVotesReliability(revision.getCurrentVotesReliability()); revision.setTotalScore(revision.getCurrentScore()); }else { double changeCoefficient = revision.getChangeCoefficient(); // Calculate total reliability of votes revision.setTotalVotesReliability(previousRevision.getTotalVotesReliability()*changeCoefficient + revision.getCurrentVotesReliability()); // Calculate total score revision.setTotalScore(previousRevision.getTotalScore() * changeCoefficient + revision.getCurrentScore()); } } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java b/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java index 5e591c9..582a0d4 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java @@ -1,242 +1,254 @@ package org.wikitolearn.wikirating.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.*; +import org.wikitolearn.wikirating.model.graph.CourseLevelThree; import org.wikitolearn.wikirating.model.graph.Process; import org.wikitolearn.wikirating.util.enums.ProcessStatus; import org.wikitolearn.wikirating.util.enums.ProcessType; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; /** * @author aletundo * @author valsdav */ @Service public class MaintenanceService { private static final Logger LOG = LoggerFactory.getLogger(MaintenanceService.class); @Autowired private UserService userService; @Autowired private PageService pageService; @Autowired private RevisionService revisionService; @Autowired private MetadataService metadataService; @Autowired private ProcessService processService; @Autowired private VoteService voteService; + @Autowired + private ComputeService computeService; @Value("#{'${mediawiki.langs}'.split(',')}") private List langs; @Value("${mediawiki.protocol}") private String protocol; @Value("${mediawiki.api.url}") private String apiUrl; @Value("${mediawiki.namespace}") private String namespace; /** * Initialize the graph for the first time using parallel threads for each domain language * @return true if initialization succeed */ public boolean initializeGraph() { // Initialize Metadata service metadataService.initMetadata(); // Start a new Process Process initProcess = processService.addProcess(ProcessType.INIT); metadataService.addFirstProcess(initProcess); CompletableFuture initFuture = CompletableFuture // Users and pages .allOf(buildUsersAndPagesFuturesList().toArray(new CompletableFuture[langs.size() + 1])) // CourseStructure .thenCompose(result -> CompletableFuture .allOf(buildApplyCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) // Revisions .thenCompose(result -> CompletableFuture .allOf(buildRevisionsFuturesList().toArray(new CompletableFuture[langs.size()]))) // Change Coefficients .thenCompose(result -> CompletableFuture .allOf(buildChangeCoefficientFuturesList().toArray(new CompletableFuture[langs.size()]))) // Users authorship .thenCompose(result -> userService.initAuthorship()); try { boolean result = initFuture.get(); // Save the result of the process if (result){ processService.closeCurrentProcess(ProcessStatus.DONE); }else{ processService.closeCurrentProcess(ProcessStatus.EXCEPTION); } return result; } catch (InterruptedException | ExecutionException e) { LOG.error("Something went wrong. {}", e.getMessage()); return false; } } /** * Entry point for the scheduled graph updated * @return true if the update succeed */ @Scheduled(cron = "${maintenance.update.cron}") public boolean updateGraph() throws UpdateGraphException{ Process currentFetchProcess; Date startTimestampCurrentFetch, startTimestampLatestFetch; // Get start timestamp of the latest FETCH Process before opening a new process startTimestampLatestFetch = (processService.getLastProcessStartDateByType(ProcessType.FETCH) != null) ? processService.getLastProcessStartDateByType(ProcessType.FETCH) : processService.getLastProcessStartDateByType(ProcessType.INIT); // Create a new FETCH process try { currentFetchProcess = processService.addProcess(ProcessType.FETCH); metadataService.updateLatestProcess(); startTimestampCurrentFetch = currentFetchProcess.getStartOfProcess(); } catch (PreviousProcessOngoingException e){ LOG.error("Cannot start Update process because the previous process is still ONGOING." + "The update will be aborted."); return false; } try { CompletableFuture updateFuture = CompletableFuture .allOf(userService.updateUsers(protocol + langs.get(0) + "." + apiUrl, startTimestampLatestFetch, startTimestampCurrentFetch)) .thenCompose(result -> CompletableFuture .allOf(buildUpdatePagesFuturesList(startTimestampLatestFetch, startTimestampCurrentFetch) .toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> CompletableFuture .allOf(buildApplyCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> voteService.validateTemporaryVotes(startTimestampCurrentFetch)); boolean result = updateFuture.get(); // Save the result of the process, closing the current one if (result) { processService.closeCurrentProcess(ProcessStatus.DONE); } else { processService.closeCurrentProcess(ProcessStatus.ERROR); } return result; } catch (TemporaryVoteValidationException | UpdateUsersException | UpdatePagesAndRevisionsException | InterruptedException | ExecutionException e) { processService.closeCurrentProcess(ProcessStatus.EXCEPTION); LOG.error("An error occurred during a scheduled graph update procedure"); throw new UpdateGraphException(); } } - + + public boolean computePageRanking(){ + List pages = pageService.getCourseLevelThreePages("es"); + for (CourseLevelThree page : pages ){ + computeService.computeRevisionsRating(page.getLangPageId()); + } + return true; + } + /** * Build a list of CompletableFuture. * * @return a list of CompletableFuture */ private List> buildApplyCourseStructureFuturesList() { List> parallelUpdateCourseStructureFutures = new ArrayList<>(); // Add course structure for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelUpdateCourseStructureFutures.add(pageService.applyCourseStructure(lang, url)); } return parallelUpdateCourseStructureFutures; } /** * * @param start * @param end * @return */ private List> buildUpdatePagesFuturesList(Date start, Date end) { List> futures = new ArrayList<>(); // Add update pages for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; futures.add(pageService.updatePages(lang, url, start, end)); } return futures; } /** * Build a list of CompletableFuture. The elements are the fetches of pages' * revisions from each domain language. * * @return a list of CompletableFuture */ private List> buildRevisionsFuturesList() { List> parallelRevisionsFutures = new ArrayList<>(); // Add revisions fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelRevisionsFutures.add(revisionService.initRevisions(lang, url)); } return parallelRevisionsFutures; } private List> buildChangeCoefficientFuturesList(){ List> parallelCCFutures = new ArrayList<>(); // Add revisions fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelCCFutures.add(revisionService.calculateChangeCoefficientAllRevisions(lang, url)); } return parallelCCFutures; } /** * Build a list of CompletableFuture. The first one is the fetch of the * users from the first domain in mediawiki.langs list. The rest of the * elements are the fetches of the pages for each language. This * implementation assumes that the users are shared among domains. * * @return a list of CompletableFuture */ private List> buildUsersAndPagesFuturesList() { List> usersAndPagesInsertion = new ArrayList<>(); // Add users fetch as fist operation usersAndPagesInsertion.add(userService.initUsers(protocol + langs.get(0) + "." + apiUrl)); // Add pages fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; usersAndPagesInsertion.add(pageService.initPages(lang, url)); } return usersAndPagesInsertion; } /*@SuppressWarnings("unchecked") private List> buildFuturesList(Object obj, String methodPrefix) { List> futures = new ArrayList<>(); for (String lang : langs) { String url = protocol + lang + "." + apiUrl; Method[] methods = obj.getClass().getMethods(); for (int i = 0; i < methods.length; i++) { try { if (methods[i].getName().startsWith(methodPrefix) && methods[i].getParameterCount() == 1) { futures.add((CompletableFuture) methods[i].invoke(obj, url)); } else if (methods[i].getName().startsWith(methodPrefix) && methods[i].getParameterCount() == 2) { futures.add((CompletableFuture) methods[i].invoke(obj, lang, url)); } } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } return futures; }*/ } diff --git a/src/main/java/org/wikitolearn/wikirating/service/PageService.java b/src/main/java/org/wikitolearn/wikirating/service/PageService.java index ba0f137..c3e03a7 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/PageService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/PageService.java @@ -1,378 +1,382 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.GetPagesUpdateInfoException; import org.wikitolearn.wikirating.exception.PageNotFoundException; import org.wikitolearn.wikirating.exception.UpdatePagesAndRevisionsException; import org.wikitolearn.wikirating.model.CourseTree; import org.wikitolearn.wikirating.model.UpdateInfo; import org.wikitolearn.wikirating.model.graph.*; import org.wikitolearn.wikirating.repository.CourseLevelThreeRepository; import org.wikitolearn.wikirating.repository.CourseLevelTwoRepository; import org.wikitolearn.wikirating.repository.CourseRootRepository; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.service.mediawiki.PageMediaWikiService; import org.wikitolearn.wikirating.service.mediawiki.UpdateMediaWikiService; import org.wikitolearn.wikirating.util.enums.CourseLevel; import com.google.common.base.CaseFormat; /** * * @author aletundo, valsdav * */ @Service public class PageService { private static final Logger LOG = LoggerFactory.getLogger(PageService.class); @Autowired private PageMediaWikiService pageMediaWikiService; @Autowired private RevisionService revisionService; @Autowired private UpdateMediaWikiService updateMediaWikiService; @Autowired private UserService userService; @Autowired private PageRepository pageRepository; @Autowired private CourseRootRepository courseRootRepository; @Autowired private CourseLevelTwoRepository courseLevelTwoRepository; @Autowired private CourseLevelThreeRepository courseLevelThreeRepository; @Value("${mediawiki.namespace}") private String namespace; /** * This methods inserts all the pages inside the DB querying the MediaWiki API. * @param lang String * @param apiUrl String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initPages( String lang, String apiUrl ){ List pages = pageMediaWikiService.getAll(apiUrl); pages.forEach(page -> { page.setLang(lang); page.setLangPageId(lang + "_" +page.getPageId()); //Now we check the level of the page to set the right // additional laber for the Course Structure. CourseLevel levelLabel = getPageLevelFromTitle(page.getTitle()); if (levelLabel != CourseLevel.UNCATEGORIZED){ String label = CaseFormat.UPPER_UNDERSCORE.to(CaseFormat.UPPER_CAMEL, levelLabel.name()); page.addLabel(label); } }); pageRepository.save(pages); LOG.info("Inserted all {} pages", lang); return CompletableFuture.completedFuture(true); } /** * * @param lang * @param apiUrl * @param start * @param end * @return * @throws UpdatePagesAndRevisionsException */ @Async public CompletableFuture updatePages(String lang, String apiUrl, Date start, Date end) throws UpdatePagesAndRevisionsException{ try{ List updates = updateMediaWikiService.getPagesUpdateInfo(apiUrl, namespace, start, end); for(UpdateInfo update : updates){ if(!namespace.equals(update.getNs())) continue; switch (update.getType()) { case NEW: // Add the new page with the right Course level Page newPage = addCoursePage(update.getPageLevelFromTitle(),update.getPageid(), update.getTitle(), lang); if (update.getPageLevelFromTitle() == CourseLevel.COURSE_LEVEL_THREE) { // Create the new revision. The change coefficient for the new revision is set to 0 by default. Revision newRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Add the first revision to the page ((CourseLevelThree) newPage).initFirstRevision(newRev); // It's necessary to save the page again courseLevelThreeRepository.save((CourseLevelThree) newPage); userService.setAuthorship(newRev); } break; case EDIT: // Act only on CourseLevelThree pages if (update.getPageLevelFromTitle() == CourseLevel.COURSE_LEVEL_THREE){ // Create a new revision Revision updateRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Then add it to the page addRevisionToPage(lang + "_" + update.getPageid(), updateRev); // Then calculate the changeCoefficient revisionService.setChangeCoefficient(apiUrl, updateRev); // Finally set the authorship userService.setAuthorship(updateRev); } break; case MOVE: // We have to change the label in case the Course level is changed // Move the page to the new title movePage(update.getTitle(), update.getNewTitle(), lang); break; case DELETE: // Delete the page and all its revisions deletePage(update.getTitle(), lang); break; default: break; } } }catch(GetPagesUpdateInfoException | PageNotFoundException e){ LOG.error("An error occurred while updating pages and revisions: {}", e.getMessage()); throw new UpdatePagesAndRevisionsException(); } return CompletableFuture.completedFuture(true); } /** * Create a new generic Page entity. * @param pageid * @param title * @param lang * @return the added page */ public Page addPage(int pageid, String title, String lang){ Page page = new Page(pageid, title, lang, lang + "_" + pageid); pageRepository.save(page); return page; } /** * Add a page entity distinguishing between the different Course levels. * @param level * @param pageid * @param title * @param lang * @return */ public Page addCoursePage(CourseLevel level, int pageid, String title, String lang){ switch(level){ case COURSE_ROOT: CourseRoot pageRoot = new CourseRoot(pageid, title, lang, lang + "_" + pageid); courseRootRepository.save(pageRoot); return pageRoot; case COURSE_LEVEL_TWO: CourseLevelTwo pageTwo = new CourseLevelTwo(pageid, title, lang, lang + "_" + pageid); courseLevelTwoRepository.save(pageTwo); return pageTwo; case COURSE_LEVEL_THREE: CourseLevelThree pageThree = new CourseLevelThree(pageid, title, lang, lang + "_" + pageid); courseLevelThreeRepository.save(pageThree); return pageThree; default: return addPage(pageid, title, lang); } } /** * Get the page with the given pageId and language * @param pageId the id of the page * @param lang the language of the page * @return the requested page * @throws PageNotFoundException */ public Page getPage(int pageId, String lang) throws PageNotFoundException{ Page page = pageRepository.findByLangPageId(lang + "_" + pageId); if(page == null){ LOG.error("Page with pageId {} and lang {} not found.", pageId, lang); throw new PageNotFoundException(); } return page; } /** * Get the page with the given langePageId * @param langPageId the langPageId of the page * @return the requested page * @throws PageNotFoundException */ public Page getPage(String langPageId) throws PageNotFoundException{ Page page = pageRepository.findByLangPageId(langPageId); if(page == null){ LOG.error("Page with langPageId: {} not found.", langPageId); throw new PageNotFoundException(); } return page; } + public List getCourseLevelThreePages(String lang) { + return courseLevelThreeRepository.findByLang(lang); + } + /** * Add a new revision to a CourseLevelThree page. It links the page to the new revision via * LAST_REVISION link. Moreover it create the PREVIOUS_REVISION link. * @param langPageId * @param rev */ public void addRevisionToPage(String langPageId, Revision rev) throws PageNotFoundException{ CourseLevelThree page = courseLevelThreeRepository.findByLangPageId(langPageId); if(page == null){ throw new PageNotFoundException(); } // Add PREVIOUS_REVISION relationship rev.setPreviousRevision(page.getLastRevision()); revisionService.updateRevision(rev); //Update the LAST_REVISION edge with a query courseLevelThreeRepository.updateLastRevision(page.getLangPageId()); } /** * Change title of a page. The method is prefixed by move to follow MediaWiki naming. * The method checks also the new page title to set the right Course level label * in case of changes * @param oldTitle the old title of the page * @param newTitle the new title to set * @param lang the language of the page * @return the updated page * @throws PageNotFoundException */ public Page movePage(String oldTitle, String newTitle, String lang) throws PageNotFoundException{ Page page = pageRepository.findByTitleAndLang(oldTitle, lang); if(page == null){ throw new PageNotFoundException(); } page.setTitle(newTitle); // Check if the Course level has changed if (getPageLevelFromTitle(oldTitle) != getPageLevelFromTitle(newTitle)){ page.removeLabel(getPageLevelFromTitle(oldTitle).name()); page.addLabel(getPageLevelFromTitle(newTitle).name()); } pageRepository.save(page); return page; } /** * Delete a page from the graph given its title and domain language * @param title the title of the page * @param lang the language of the page * @throws PageNotFoundException */ public void deletePage(String title, String lang) throws PageNotFoundException{ Page page = pageRepository.findByTitleAndLang(title, lang); if(page == null){ throw new PageNotFoundException(); } // Delete the revisions of the page if it's CourseLevelThree if (page.hasLabel("CourseLevelThree")){ revisionService.deleteRevisionsOfPage(page.getLangPageId()); } // Delete finally the page itself pageRepository.delete(page); } /** * Get the level of a Page in the Course Structure * analyzing the number of slashes in the title * @param title the title of the page * @return the course level */ public static CourseLevel getPageLevelFromTitle(String title){ int slashesNumber = StringUtils.countMatches(title, "/"); switch (slashesNumber){ case 0: return CourseLevel.COURSE_ROOT; case 1: return CourseLevel.COURSE_LEVEL_TWO; case 2: return CourseLevel.COURSE_LEVEL_THREE; default: //The page remains generic. return CourseLevel.UNCATEGORIZED; } } /** * Get the pages labeled by :CourseRoot label * @return the list of course root pages */ public List getCourseRootPages(String lang){ return courseRootRepository.findByLang(lang); } /** * Get the page labeled only by :Page label * @return the list of the uncategorized pages */ public List getUncategorizedPages(String lang){ return pageRepository.findAllUncategorizedPages(lang); } /** * @param lang the language of the domain * @param apiUrl the MediaWiki API url - * @param courseRootPages */ @Async public CompletableFuture applyCourseStructure(String lang, String apiUrl) { List courseRootPages = getCourseRootPages(lang); for (CourseRoot pageRoot : courseRootPages) { // Get course tree and prepare relationship set CourseTree tree = pageMediaWikiService.getCourseTree(apiUrl, pageRoot.getTitle()); Set levelsTwo = (pageRoot.getLevelsTwo() == null) ? new HashSet<>() : pageRoot.getLevelsTwo(); int index = 0; for (String levelTwo : tree.getLevelsTwo()) { String levelTwoTitle = (tree.getRoot() + "/" + levelTwo).trim(); CourseLevelTwo levelTwoPage = courseLevelTwoRepository.findByTitleAndLang(levelTwoTitle, lang); // Skip malformed page if (levelTwoPage == null) continue; // Add levels two to the set to be saved if(!levelsTwo.contains(levelTwoPage)){ levelsTwo.add(levelTwoPage); } Set levelsThree = (levelTwoPage.getLevelsThree() == null) ? new HashSet<>() : levelTwoPage.getLevelsThree(); // Add levels three to the set to be saved for (String levelThree : tree.getLevelsTree().get(index)) { String levelThreeTitle = (levelTwoTitle + "/" + levelThree).trim(); CourseLevelThree levelThreePage = courseLevelThreeRepository.findByTitleAndLang(levelThreeTitle, lang); // Skip malformed page if (levelThreePage == null) continue; if(!levelsThree.contains(levelThreePage)){ levelsThree.add(levelThreePage); } } // Set LEVEL_THREE relationships levelTwoPage.setLevelsThree(levelsThree); courseLevelThreeRepository.save(levelsThree); index++; } // Set LEVEL_TWO relationships pageRoot.setLevelsTwo(levelsTwo); courseLevelTwoRepository.save(levelsTwo); courseRootRepository.save(pageRoot); } return CompletableFuture.completedFuture(true); } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index 73b70fb..ecd5178 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,223 +1,257 @@ /** * */ package org.wikitolearn.wikirating.service; -import java.util.Date; -import java.util.List; -import java.util.ListIterator; -import java.util.Set; +import java.util.*; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.model.graph.CourseLevelThree; import org.wikitolearn.wikirating.model.graph.Revision; +import org.wikitolearn.wikirating.model.graph.queryresult.RevisionResult; import org.wikitolearn.wikirating.repository.CourseLevelThreeRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; import static java.lang.Math.exp; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private CourseLevelThreeRepository courseLevelThreeRepository; /** * Initialize the revisions for the first time querying the MediaWiki API. * This method adds the revisions and sets the FIRST_REVISION, * LAST_REVISION and PREVIOUS_REVISION relationships. * Only the revisions of CourseLevelThree pages are fetched. * @param lang the domain language * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initRevisions(String lang, String apiUrl) { List pages = courseLevelThreeRepository.findByLang(lang); for(CourseLevelThree page : pages){ List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageId()); // Set the first and the last revisions for the current page page.setFirstRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); // Set the last validated revision to the first one. page.setLastValidatedRevision(revisions.get(0)); // Create the PreviousRevision links ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevId()); rev.setLang(lang); if (it.previousIndex() != 0) { rev.setPreviousRevision(revisions.get(it.previousIndex() - 1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); courseLevelThreeRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); } return CompletableFuture.completedFuture(true); } @Async public CompletableFuture calculateChangeCoefficientAllRevisions(String lang, String apiUrl){ LOG.info("Calculating changeCoefficient for all the revisions..."); Set revisions = revisionRepository.findByLang(lang); for (Revision rev : revisions){ double changeCoefficient = calculateChangeCoefficient(apiUrl, rev); rev.setChangeCoefficient(changeCoefficient); } revisionRepository.save(revisions); LOG.info("ChangeCoefficient calculated for all revisions."); return CompletableFuture.completedFuture(true); } /** * Add a new Revision to the graph. This method DOESN'T link the * revision to a Page. * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang, userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } /** * Delete a revision given its langPageId * @param langPageId the langPageId of the revision */ public void deleteRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } revisionRepository.delete(revisions); } public Set getRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } return revisions; } - public Set getRevisionsOfPageOrdered(String langPageId) throws RevisionNotFoundException{ - Set revisions = revisionRepository.findAllRevisionOfPageOrdered(langPageId); + /** + * Get the revisions of a page ordered by revId from the oldest to the newest. + * @param langPageId + * @return + * @throws RevisionNotFoundException + */ + public List getRevisionsOfPageOrdered(String langPageId) throws RevisionNotFoundException{ + List revisions = revisionRepository.findAllRevisionOfPageOrdered(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } return revisions; } + /** + * Get all the revisions of a page from the LAST_VALIDATED_REVISION (included) to the + * newest one ordered by revId. + * @param langPageId + * @return + * @throws RevisionNotFoundException + */ + public List getRevisionsFromLastValidated(String langPageId) throws RevisionNotFoundException{ + //List revisions = revisionRepository.findAllRevisionNotValidated(langPageId); + List revisions = new ArrayList<>(); + // We want also to add the latest validated revision to re-validate it. + // We put it at the beginning of the list because they are ordered from the oldest. + RevisionResult s = revisionRepository.getRev("es_196"); + RevisionResult r = revisionRepository.findLastValidatedRevision(langPageId); + //revisions.add(0, ); + /*if (revisions.size() == 0){ + LOG.error("Revisions of page {} not found", langPageId); + throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); + } + List result = new ArrayList<>(); + for (RevisionResult rev : revisions){ + //result.add(rev.revision); + }*/ + return result; + } + /** * Get the requested revision * @param langRevId the langRevId of the revision * @return the revision * @throws RevisionNotFoundException */ public Revision getRevision(String langRevId) throws RevisionNotFoundException{ Revision revision = revisionRepository.findByLangRevId(langRevId); if(revision == null){ LOG.error("Revision {} not found", langRevId); throw new RevisionNotFoundException(); } return revision; } /** * Update the given revision * @param revision * @return the updated revision */ public Revision updateRevision(Revision revision){ revisionRepository.save(revision); return revision; } + public void updateRevisions(Collection revisions){ + revisionRepository.save(revisions); + } + /** * Calculate and set the changeCoefficient of a Revision. * This method also persist the Revision changes. * @param apiUrl * @param revision */ public void setChangeCoefficient(String apiUrl, Revision revision){ double cc = calculateChangeCoefficient(apiUrl, revision); revision.setChangeCoefficient(cc); revisionRepository.save(revision); } /** * Calculate the changeCoefficient for a given Revision querying mediawiki * and returns the number. It doesn't store it in the Revision. * @param revision * @return */ public double calculateChangeCoefficient(String apiUrl, Revision revision){ double previousLength = 0.0; double changeCoefficient = 0.0; // Get the previous Revision Revision previousRevision = revision.getPreviousRevision(); if (previousRevision == null){ previousLength = 1.0; changeCoefficient = 0.0; LOG.info("Change coefficient of revision {} (first-rev): {}", revision.getLangRevId(), changeCoefficient); } else{ double prevL = (double) previousRevision.getLength(); // Suppose the mean line length of 120 characters and that the length is in bytes. // We want a "lenght" in n° of lines if (prevL == 0){ previousLength = 1.0; }else { previousLength = (prevL < 120) ? 1 : prevL / 120; } // Query mediawiki for diff text String diffText = revisionMediaWikiService.getDiffPreviousRevision(apiUrl, previousRevision.getRevId(), revision.getRevId()); int addedLines = StringUtils.countMatches(diffText, "diff-addedline"); int deletedLines = StringUtils.countMatches(diffText, "diff-deletedline"); //int inlineChanges = StringUtils.countMatches(diffText, "diffchange-inline"); // Finally calculation of change Coefficient double t = ((1.2 * deletedLines + addedLines) ) / previousLength; changeCoefficient = 1 / exp(0.5 * t); LOG.info("Change coefficient of revision {} (+{}-{}/{}): {}", revision.getLangRevId(), addedLines, deletedLines, previousLength, changeCoefficient); } return changeCoefficient; } } diff --git a/src/main/resources/application.example.properties b/src/main/resources/application.example.properties index 0a26114..fe5a950 100644 --- a/src/main/resources/application.example.properties +++ b/src/main/resources/application.example.properties @@ -1,66 +1,68 @@ # ---------------------------------------- # The following properties values are for example purposes and are not intended # to be used in production. # Please, copy the content in application.properties file or where you prefer # and change the values. # ---------------------------------------- # ---------------------------------------- # DATABASE PROPERTIES # ---------------------------------------- spring.data.neo4j.uri=bolt://localhost:7687 spring.data.neo4j.username=neo4j spring.data.neo4j.password=neo4j # ---------------------------------------- # MAINTENANCE PROPERTIES # ---------------------------------------- maintenance.path=/maintenance maintenance.readonlymode.uri=${maintenance.path}/read_only maintenance.init.uri=${maintenance.path}/init maintenance.fetch.uri=${maintenance.path}/fetch +maintenance.compute.uri=${maintenance.path}/compute maintenance.wipe.uri=${maintenance.path}/wipe maintenance.update.cron=0 0/30 * * * ? +maintenance.compute.cron= 0 0/40 * * * ? # ---------------------------------------- # SECURITY PROPERTIES # ---------------------------------------- security.basic.authorize-mode=role security.basic.realm=Maintenance security.basic.path=${maintenance.path}/** security.user.name=admin security.user.role=ADMIN security.user.password=secret security.headers.xss=true # ---------------------------------------- # MANAGEMENT PROPERTIES # ---------------------------------------- management.port=8081 management.security.roles=ADMIN # ---------------------------------------- # INFO PROPERTIES # ---------------------------------------- info.app.name=WikiRating info.app.description=A Spring Boot application that relies on Neo4j to serve a rating engine for a MediaWiki installation info.app.version=0.0.1 # ---------------------------------------- # ENDPOINT PROPERTIES # ---------------------------------------- endpoints.enabled=false endpoints.info.enabled=true endpoints.health.enabled=true endpoints.metrics.enabled=true endpoints.trace.enabled=true # ---------------------------------------- # MEDIAWIKI PROPERTIES # ---------------------------------------- mediawiki.langs=your,domains,languages,list mediawiki.protocol=http:// or https:// mediawiki.api.url=your_api_url mediawiki.api.user=user mediawiki.api.password=secret mediawiki.namespace=0 \ No newline at end of file