diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java b/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java index f5de9d2..fca86f6 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java @@ -1,375 +1,360 @@ package org.wikitolearn.wikirating.model.graph; import org.neo4j.ogm.annotation.GraphId; import org.neo4j.ogm.annotation.Index; import org.neo4j.ogm.annotation.NodeEntity; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import org.neo4j.ogm.annotation.Relationship; import org.neo4j.ogm.annotation.typeconversion.DateLong; //import org.springframework.data.annotation.Transient; import java.util.Date; import java.util.Set; /** * This class handles the data of the Revision of a page. * Created by valsdav on 14/03/17. */ @JsonIgnoreProperties(ignoreUnknown = true) @NodeEntity( label = "Revision" ) public class Revision { @GraphId @JsonIgnore private Long graphId; + @Index(unique = true, primary=true) + @JsonProperty(access = Access.WRITE_ONLY) + private String langRevId; @JsonProperty("revid") private int revId; @JsonProperty(access = Access.WRITE_ONLY) private String lang; @JsonProperty(value = "userid", access = Access.WRITE_ONLY) private int userId; @JsonProperty(value = "parentid", access = Access.WRITE_ONLY) private int parentId; @DateLong @JsonProperty(access = Access.WRITE_ONLY) private Date timestamp; @JsonProperty(value = "size", access = Access.WRITE_ONLY) private long length; @JsonProperty(access = Access.WRITE_ONLY) private double changeCoefficient; private double currentMeanVote; private double currentVotesReliability; - private double currentNormalisesVotesReliability; private double totalMeanVote; private double totalVotesReliability; - private double totalNormalisesVotesReliability; - @JsonProperty(access = Access.WRITE_ONLY) - private boolean validated; - @Index(unique = true, primary=true) - @JsonProperty(access = Access.WRITE_ONLY) - private String langRevId; + private double normalizedNumberOfVotes; + private double currentScore; + private double totalScore; + @Relationship(type="PREVIOUS_REVISION", direction = Relationship.OUTGOING) private Revision previousRevision; @Relationship(type="VOTE", direction = Relationship.INCOMING) private Set votes; //This relationship represents the connection between the revision //and its creator (user). It saves his reliability at the moment of the author. @Relationship(type="AUTHOR", direction = Relationship.INCOMING) private Author author; /** * */ public Revision() {} /** * @param revId * @param userId * @param parentId * @param length * @param lang * @param timestamp */ public Revision(int revId, String lang, int userId, int parentId, long length, Date timestamp) { this.revId = revId; this.langRevId = lang +"_"+revId; this.userId = userId; this.parentId = parentId; this.length = length; this.lang = lang; this.timestamp = timestamp; - this.validated = false; this.changeCoefficient = 0.0; } /** * @return the revId */ public int getRevId() { return revId; } /** * @param revId the revId to set */ public void setRevId(int revId) { this.revId = revId; } /** * @return the userId */ public int getUserId() { return userId; } /** - * @param userid the userId to set + * @param userId the userId to set */ public void setUserId(int userId) { this.userId = userId; } /** * @return the parentId */ public int getParentId() { return parentId; } /** * @param parentId the parentId to set */ public void setParentId(int parentId) { this.parentId = parentId; } /** * @return the length */ public long getLength() { return length; } /** * @param length the length to set */ public void setLength(long length) { this.length = length; } /** * @return the changeCoefficient */ public double getChangeCoefficient() { return changeCoefficient; } /** * @param changeCoefficient the changeCoefficient to set */ public void setChangeCoefficient(double changeCoefficient) { this.changeCoefficient = changeCoefficient; } /** * @return the currentMeanVote */ public double getCurrentMeanVote() { return currentMeanVote; } /** * @param currentMeanVote the currentMeanVote to set */ public void setCurrentMeanVote(double currentMeanVote) { this.currentMeanVote = currentMeanVote; } /** * @return the currentVotesReliability */ public double getCurrentVotesReliability() { return currentVotesReliability; } /** * @param currentVotesReliability the currentVotesReliability to set */ public void setCurrentVotesReliability(double currentVotesReliability) { this.currentVotesReliability = currentVotesReliability; } - /** - * @return the currentNormalisesVotesReliability - */ - public double getCurrentNormalisesVotesReliability() { - return currentNormalisesVotesReliability; - } - - /** - * @param currentNormalisesVotesReliability the currentNormalisesVotesReliability to set - */ - public void setCurrentNormalisesVotesReliability(double currentNormalisesVotesReliability) { - this.currentNormalisesVotesReliability = currentNormalisesVotesReliability; - } - /** * @return the totalMeanVote */ public double getTotalMeanVote() { return totalMeanVote; } /** * @param totalMeanVote the totalMeanVote to set */ public void setTotalMeanVote(double totalMeanVote) { this.totalMeanVote = totalMeanVote; } /** * @return the totalVotesReliability */ public double getTotalVotesReliability() { return totalVotesReliability; } /** * @param totalVotesReliability the totalVotesReliability to set */ public void setTotalVotesReliability(double totalVotesReliability) { this.totalVotesReliability = totalVotesReliability; } - /** - * @return the totalNormalisesVotesReliability - */ - public double getTotalNormalisesVotesReliability() { - return totalNormalisesVotesReliability; - } + public double getCurrentScore() { + return currentScore; + } - /** - * @param totalNormalisesVotesReliability the totalNormalisesVotesReliability to set - */ - public void setTotalNormalisesVotesReliability(double totalNormalisesVotesReliability) { - this.totalNormalisesVotesReliability = totalNormalisesVotesReliability; - } + public void setCurrentScore(double currentScore) { + this.currentScore = currentScore; + } - /** - * @return the validated - */ - public boolean isValidated() { - return validated; - } + public double getTotalScore() { + return totalScore; + } - /** - * @param validated the validated to set - */ - public void setValidated(boolean validated) { - this.validated = validated; - } + public void setTotalScore(double totalScore) { + this.totalScore = totalScore; + } - /** + /** * @return the lang */ public String getLang() { return lang; } /** * @param lang the lang to set */ public void setLang(String lang) { this.lang = lang; } /** * @return the langRevId */ public String getLangRevId() { return langRevId; } /** * @param langRevId the langRevId to set */ public void setLangRevId(String langRevId) { this.langRevId = langRevId; } /** * @return the graphId */ public Long getGraphId() { return graphId; } /** * @param graphId the graphId to set */ public void setGraphId(Long graphId) { this.graphId = graphId; } /** * @return the timestamp */ public Date getTimestamp() { return timestamp; } /** * @param timestamp the timestamp to set */ public void setTimestamp(Date timestamp) { this.timestamp = timestamp; } /** * @return the previousRevision */ public Revision getPreviousRevision() { return previousRevision; } /** * @param previousRevision the previousRevision to set */ public void setPreviousRevision(Revision previousRevision) { this.previousRevision = previousRevision; } /** * @return the votes */ public Set getVotes() { return votes; } /** * @param votes the votes to set */ public void setVotes(Set votes) { this.votes = votes; } public Author getAuthor() { return author; } public void setAuthor(Author author) { this.author = author; } + public double getNormalizedNumberOfVotes() { + return normalizedNumberOfVotes; + } + + public void setNormalizedNumberOfVotes(double normalizedNumberOfVotes) { + this.normalizedNumberOfVotes = normalizedNumberOfVotes; + } + + public int getCurrentNumberOfVotes() { + if (votes != null) { + return this.votes.size(); + }else{ + return 0; + } + } + /** * * @param vote the vote to set */ public void addVote(Vote vote){ this.votes.add(vote); } /* (non-Javadoc) * @see java.lang.Object#toString() */ @Override public String toString() { return "Revision{" + - "revId=" + revId + - ", lang='" + lang + '\'' + + "langRevId=" + langRevId + ", userId=" + userId + - ", parentId=" + parentId + ", timestamp=" + timestamp + ", length=" + length + - ", validated=" + validated + - ", langRevId='" + langRevId + '\'' + ", author=" + author + '}'; } } \ No newline at end of file diff --git a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java index e67db64..5a8946f 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java @@ -1,56 +1,65 @@ /** * */ package org.wikitolearn.wikirating.repository; import java.util.Set; import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.repository.GraphRepository; import org.wikitolearn.wikirating.model.graph.Revision; /** * @author aletundo * */ public interface RevisionRepository extends GraphRepository { /** * * @param langRevId * @return */ Revision findByLangRevId(String langRevId); /** * * @param lang * @return */ Set findByLang(String lang); /** * * @param userId * @return */ Set findByUserId(int userId); /** * This query returns all the Revisions of a Page (CourseLevelThree). * The direction -> of the link is important to traverse * only the chain of Revisions of the page without reaching other nodes. * @param langPageId * @return */ @Query("MATCH (p:CourseLevelThree {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r") Set findAllRevisionOfPage(String langPageId); + /** + * This query returns all the Revisions of a Page ordere from the oldest + * to the newest. + * @param langPageId + * @return + */ + @Query("MATCH (p:CourseLevelThree {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r ORDER BY r.revId") + Set findAllRevisionOfPageOrdered(String langPageId); + /** * This query return the previous revision of a Revision identified * by langRevId. * @param langRevId langRevId of the requested revision * @return the previous revision */ @Query("MATCH (r:Revision {langRevId:{0}})-[:PREVIOUS_REVISION]->(a:Revision) RETURN a") Revision findPreviousRevision(String langRevId); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/ComputeService.java b/src/main/java/org/wikitolearn/wikirating/service/ComputeService.java new file mode 100644 index 0000000..205a077 --- /dev/null +++ b/src/main/java/org/wikitolearn/wikirating/service/ComputeService.java @@ -0,0 +1,104 @@ +package org.wikitolearn.wikirating.service; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.wikitolearn.wikirating.model.graph.Revision; +import org.wikitolearn.wikirating.model.graph.Vote; + +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +/** + * Created by valsdav on 07/05/17. + */ + +@Service +public class ComputeService { + + @Autowired PageService pageService; + @Autowired RevisionService revisionService; + @Autowired UserService userService; + @Autowired VoteService voteService; + + @Async + public CompletableFuture calculateRevisionsPoints(String langPageId){ + // Get all the revision ordered from the oldest + Set revisions = revisionService.getRevisionsOfPageOrdered(langPageId); + // The order of the revision ensures that we have always the values + // for the previous revision. + for (Revision revision : revisions){ + calculateVotesMean(revision); + calculateTotalVotesMean(revision); + calculateSigmaAndScore(revision); + calculateTotalSigmaAndScore(revision); + } + return CompletableFuture.completedFuture(true); + } + + private void calculateVotesMean(Revision revision){ + double total = 0; + double weights = 0; + for (Vote vote : revision.getVotes()){ + double reliabilitySquared = Math.pow(vote.getReliability(),2); + total += reliabilitySquared * vote.getValue(); + weights += reliabilitySquared; + } + double currentMean = total/weights; + revision.setCurrentMeanVote(currentMean); + } + + private void calculateTotalVotesMean(Revision revision){ + Revision previousRevision = revision.getPreviousRevision(); + if (previousRevision == null){ + // First revision of the page + revision.setTotalMeanVote(revision.getCurrentMeanVote()); + }else { + double changeCoefficient = revision.getChangeCoefficient(); + // Calculate total mean + double previousNVotes = previousRevision.getNormalizedNumberOfVotes(); + int currentNVotes = revision.getCurrentNumberOfVotes(); + + double total = (previousNVotes * changeCoefficient) * previousRevision.getTotalMeanVote(); + total += currentNVotes * revision.getCurrentMeanVote(); + double weights = previousNVotes * changeCoefficient + currentNVotes; + revision.setTotalMeanVote(total/weights); + + // Calculate normalized number of votes + revision.setNormalizedNumberOfVotes(previousNVotes * changeCoefficient + currentNVotes); + } + } + + private void calculateSigmaAndScore(Revision revision){ + double totalMean = revision.getTotalMeanVote(); + double sigmaTotal = 0; + double score = 0; + for (Vote vote : revision.getVotes()){ + double deltaVote = totalMean - vote.getValue(); + double sigma = Math.pow(vote.getReliability(),2)* (1 - Math.pow(deltaVote,2)); + sigmaTotal += sigma; + score += sigma * Math.pow(vote.getValue(), 2); + } + revision.setCurrentVotesReliability(sigmaTotal); + revision.setCurrentScore(score); + } + + + private void calculateTotalSigmaAndScore(Revision revision){ + Revision previousRevision = revision.getPreviousRevision(); + if (previousRevision == null){ + // First revision of the page + revision.setTotalVotesReliability(revision.getCurrentVotesReliability()); + revision.setTotalScore(revision.getCurrentScore()); + }else { + double changeCoefficient = revision.getChangeCoefficient(); + // Calculate total reliability of votes + revision.setTotalVotesReliability(previousRevision.getTotalVotesReliability()*changeCoefficient + + revision.getCurrentVotesReliability()); + // Calculate total score + revision.setTotalScore(previousRevision.getTotalScore() * changeCoefficient + + revision.getCurrentScore()); + } + } + +} diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index 192dc31..73b70fb 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,214 +1,223 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.List; import java.util.ListIterator; import java.util.Set; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.model.graph.CourseLevelThree; import org.wikitolearn.wikirating.model.graph.Revision; import org.wikitolearn.wikirating.repository.CourseLevelThreeRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; import static java.lang.Math.exp; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private CourseLevelThreeRepository courseLevelThreeRepository; /** * Initialize the revisions for the first time querying the MediaWiki API. * This method adds the revisions and sets the FIRST_REVISION, * LAST_REVISION and PREVIOUS_REVISION relationships. * Only the revisions of CourseLevelThree pages are fetched. * @param lang the domain language * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initRevisions(String lang, String apiUrl) { List pages = courseLevelThreeRepository.findByLang(lang); for(CourseLevelThree page : pages){ List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageId()); // Set the first and the last revisions for the current page page.setFirstRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); // Set the last validated revision to the first one. page.setLastValidatedRevision(revisions.get(0)); // Create the PreviousRevision links ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevId()); rev.setLang(lang); if (it.previousIndex() != 0) { rev.setPreviousRevision(revisions.get(it.previousIndex() - 1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); courseLevelThreeRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); } return CompletableFuture.completedFuture(true); } @Async public CompletableFuture calculateChangeCoefficientAllRevisions(String lang, String apiUrl){ LOG.info("Calculating changeCoefficient for all the revisions..."); Set revisions = revisionRepository.findByLang(lang); for (Revision rev : revisions){ double changeCoefficient = calculateChangeCoefficient(apiUrl, rev); rev.setChangeCoefficient(changeCoefficient); } revisionRepository.save(revisions); LOG.info("ChangeCoefficient calculated for all revisions."); return CompletableFuture.completedFuture(true); } /** * Add a new Revision to the graph. This method DOESN'T link the * revision to a Page. * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang, userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } /** * Delete a revision given its langPageId * @param langPageId the langPageId of the revision */ public void deleteRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } revisionRepository.delete(revisions); } public Set getRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } return revisions; } - + + public Set getRevisionsOfPageOrdered(String langPageId) throws RevisionNotFoundException{ + Set revisions = revisionRepository.findAllRevisionOfPageOrdered(langPageId); + if (revisions.size() == 0){ + LOG.error("Revisions of page {} not found", langPageId); + throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); + } + return revisions; + } + /** * Get the requested revision * @param langRevId the langRevId of the revision * @return the revision * @throws RevisionNotFoundException */ public Revision getRevision(String langRevId) throws RevisionNotFoundException{ Revision revision = revisionRepository.findByLangRevId(langRevId); if(revision == null){ LOG.error("Revision {} not found", langRevId); throw new RevisionNotFoundException(); } return revision; } /** * Update the given revision * @param revision * @return the updated revision */ public Revision updateRevision(Revision revision){ revisionRepository.save(revision); return revision; } /** * Calculate and set the changeCoefficient of a Revision. * This method also persist the Revision changes. * @param apiUrl * @param revision */ public void setChangeCoefficient(String apiUrl, Revision revision){ double cc = calculateChangeCoefficient(apiUrl, revision); revision.setChangeCoefficient(cc); revisionRepository.save(revision); } /** * Calculate the changeCoefficient for a given Revision querying mediawiki * and returns the number. It doesn't store it in the Revision. * @param revision * @return */ public double calculateChangeCoefficient(String apiUrl, Revision revision){ double previousLength = 0.0; double changeCoefficient = 0.0; // Get the previous Revision Revision previousRevision = revision.getPreviousRevision(); if (previousRevision == null){ previousLength = 1.0; changeCoefficient = 0.0; LOG.info("Change coefficient of revision {} (first-rev): {}", revision.getLangRevId(), changeCoefficient); } else{ double prevL = (double) previousRevision.getLength(); // Suppose the mean line length of 120 characters and that the length is in bytes. // We want a "lenght" in n° of lines if (prevL == 0){ previousLength = 1.0; }else { previousLength = (prevL < 120) ? 1 : prevL / 120; } // Query mediawiki for diff text String diffText = revisionMediaWikiService.getDiffPreviousRevision(apiUrl, previousRevision.getRevId(), revision.getRevId()); int addedLines = StringUtils.countMatches(diffText, "diff-addedline"); int deletedLines = StringUtils.countMatches(diffText, "diff-deletedline"); //int inlineChanges = StringUtils.countMatches(diffText, "diffchange-inline"); // Finally calculation of change Coefficient double t = ((1.2 * deletedLines + addedLines) ) / previousLength; changeCoefficient = 1 / exp(0.5 * t); LOG.info("Change coefficient of revision {} (+{}-{}/{}): {}", revision.getLangRevId(), addedLines, deletedLines, previousLength, changeCoefficient); } return changeCoefficient; } }