diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/Author.java b/src/main/java/org/wikitolearn/wikirating/model/graph/Author.java index 68b268f..00c6c75 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/graph/Author.java +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/Author.java @@ -1,101 +1,101 @@ package org.wikitolearn.wikirating.model.graph; import org.neo4j.ogm.annotation.EndNode; import org.neo4j.ogm.annotation.GraphId; import org.neo4j.ogm.annotation.RelationshipEntity; import org.neo4j.ogm.annotation.StartNode; import com.fasterxml.jackson.annotation.JsonIgnore; /** * This entity represent the edit of a user on a Page. * It stores the the reliability of the user at the moment * of the creation. */ -@RelationshipEntity(type="Author") +@RelationshipEntity(type="AUTHOR") public class Author { @GraphId @JsonIgnore private Long graphId; private double reliability; @StartNode private User user; @EndNode private Revision revision; /** * */ public Author() {} /** * @param reliability */ public Author(double reliability) { this.reliability = reliability; } /** * @return the graphId */ public Long getGraphId() { return graphId; } /** * @param graphId the graphId to set */ public void setGraphId(Long graphId) { this.graphId = graphId; } /** * @return the reliability */ public double getReliability() { return reliability; } /** * @param reliability the reliability to set */ public void setReliability(double reliability) { this.reliability = reliability; } /** * @return the user */ public User getUser() { return user; } /** * @param user the user to set */ public void setUser(User user) { this.user = user; } /** * @return the revision */ public Revision getRevision() { return revision; } /** * @param revision the revision to set */ public void setRevision(Revision revision) { this.revision = revision; } /* (non-Javadoc) * @see java.lang.Object#toString() */ @Override public String toString() { return "Author [graphId=" + graphId + ", reliability=" + reliability + ", user=" + user + ", revision=" + revision + "]"; } } diff --git a/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java b/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java index ebdffb6..d630547 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java +++ b/src/main/java/org/wikitolearn/wikirating/model/graph/Revision.java @@ -1,374 +1,388 @@ package org.wikitolearn.wikirating.model.graph; import org.neo4j.ogm.annotation.GraphId; import org.neo4j.ogm.annotation.Index; import org.neo4j.ogm.annotation.NodeEntity; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonProperty.Access; import org.neo4j.ogm.annotation.Relationship; import org.neo4j.ogm.annotation.typeconversion.DateLong; //import org.springframework.data.annotation.Transient; import java.util.Date; +import java.util.HashSet; import java.util.Set; /** * This class handles the data of the Revision of a page. * Created by valsdav on 14/03/17. */ @JsonIgnoreProperties(ignoreUnknown = true) @NodeEntity( label = "Revision" ) public class Revision { @GraphId @JsonIgnore private Long graphId; @Index(unique = true, primary=true) @JsonProperty(access = Access.WRITE_ONLY) private String langRevId; @JsonProperty("revid") private int revId; @JsonProperty(access = Access.WRITE_ONLY) private String lang; @JsonProperty(value = "userid", access = Access.WRITE_ONLY) private int userId; @JsonProperty(value = "parentid", access = Access.WRITE_ONLY) private int parentId; @DateLong @JsonProperty(access = Access.WRITE_ONLY) private Date timestamp; @JsonProperty(value = "size", access = Access.WRITE_ONLY) private long length; @JsonProperty(access = Access.WRITE_ONLY) private double changeCoefficient; private double currentMeanVote; private double currentVotesReliability; private double totalMeanVote; private double totalVotesReliability; private double normalizedNumberOfVotes; private double currentScore; private double totalScore; @Relationship(type="PREVIOUS_REVISION", direction = Relationship.OUTGOING) private Revision previousRevision; - //@Relationship(type="PREVISOU_REVISION", direction = Relationship.INCOMING) - //private Revision nextRevision; + @Relationship(type="PREVIOUS_REVISION", direction = Relationship.INCOMING) + private Revision nextRevision; @Relationship(type="VOTE", direction = Relationship.INCOMING) private Set votes; //This relationship represents the connection between the revision //and its creator (user). It saves his reliability at the moment of the author. @Relationship(type="AUTHOR", direction = Relationship.INCOMING) private Author author; /** * */ public Revision() {} /** * @param revId * @param userId * @param parentId * @param length * @param lang * @param timestamp */ public Revision(int revId, String lang, int userId, int parentId, long length, Date timestamp) { this.revId = revId; this.langRevId = lang +"_"+revId; this.userId = userId; this.parentId = parentId; this.length = length; this.lang = lang; this.timestamp = timestamp; this.changeCoefficient = 0.0; } /** * @return the revId */ public int getRevId() { return revId; } /** * @param revId the revId to set */ public void setRevId(int revId) { this.revId = revId; } /** * @return the userId */ public int getUserId() { return userId; } /** * @param userId the userId to set */ public void setUserId(int userId) { this.userId = userId; } /** * @return the parentId */ public int getParentId() { return parentId; } /** * @param parentId the parentId to set */ public void setParentId(int parentId) { this.parentId = parentId; } /** * @return the length */ public long getLength() { return length; } /** * @param length the length to set */ public void setLength(long length) { this.length = length; } /** * @return the changeCoefficient */ public double getChangeCoefficient() { return changeCoefficient; } /** * @param changeCoefficient the changeCoefficient to set */ public void setChangeCoefficient(double changeCoefficient) { this.changeCoefficient = changeCoefficient; } /** * @return the currentMeanVote */ public double getCurrentMeanVote() { return currentMeanVote; } /** * @param currentMeanVote the currentMeanVote to set */ public void setCurrentMeanVote(double currentMeanVote) { this.currentMeanVote = currentMeanVote; } /** * @return the currentVotesReliability */ public double getCurrentVotesReliability() { return currentVotesReliability; } /** * @param currentVotesReliability the currentVotesReliability to set */ public void setCurrentVotesReliability(double currentVotesReliability) { this.currentVotesReliability = currentVotesReliability; } /** * @return the totalMeanVote */ public double getTotalMeanVote() { return totalMeanVote; } /** * @param totalMeanVote the totalMeanVote to set */ public void setTotalMeanVote(double totalMeanVote) { this.totalMeanVote = totalMeanVote; } /** * @return the totalVotesReliability */ public double getTotalVotesReliability() { return totalVotesReliability; } /** * @param totalVotesReliability the totalVotesReliability to set */ public void setTotalVotesReliability(double totalVotesReliability) { this.totalVotesReliability = totalVotesReliability; } public double getCurrentScore() { return currentScore; } public void setCurrentScore(double currentScore) { this.currentScore = currentScore; } public double getTotalScore() { return totalScore; } public void setTotalScore(double totalScore) { this.totalScore = totalScore; } /** * @return the lang */ public String getLang() { return lang; } /** * @param lang the lang to set */ public void setLang(String lang) { this.lang = lang; } /** * @return the langRevId */ public String getLangRevId() { return langRevId; } /** * @param langRevId the langRevId to set */ public void setLangRevId(String langRevId) { this.langRevId = langRevId; } /** * @return the graphId */ public Long getGraphId() { return graphId; } /** * @param graphId the graphId to set */ public void setGraphId(Long graphId) { this.graphId = graphId; } /** * @return the timestamp */ public Date getTimestamp() { return timestamp; } /** * @param timestamp the timestamp to set */ public void setTimestamp(Date timestamp) { this.timestamp = timestamp; } /** * @return the previousRevision */ public Revision getPreviousRevision() { return previousRevision; } /** * @param previousRevision the previousRevision to set */ public void setPreviousRevision(Revision previousRevision) { this.previousRevision = previousRevision; } - /* + public boolean hasPreviousRevision(){ + return this.nextRevision != null; + } + public Revision getNextRevision() { return nextRevision; } public void setNextRevision(Revision nextRevision) { this.nextRevision = nextRevision; } -*/ + + public boolean hasNextRevision(){ + return this.nextRevision != null; + } /** * @return the votes */ public Set getVotes() { + if (votes == null){ + return new HashSet<>(); + } return votes; } /** * @param votes the votes to set */ public void setVotes(Set votes) { this.votes = votes; } public Author getAuthor() { return author; } public void setAuthor(Author author) { this.author = author; } public double getNormalizedNumberOfVotes() { return normalizedNumberOfVotes; } public void setNormalizedNumberOfVotes(double normalizedNumberOfVotes) { this.normalizedNumberOfVotes = normalizedNumberOfVotes; } public int getCurrentNumberOfVotes() { if (votes != null) { return this.votes.size(); }else{ return 0; } } /** * * @param vote the vote to set */ public void addVote(Vote vote){ this.votes.add(vote); } /* (non-Javadoc) * @see java.lang.Object#toString() */ @Override public String toString() { - return "Revision{" + + String s= "Revision{" + "langRevId=" + langRevId + ", userId=" + userId + ", timestamp=" + timestamp + - ", length=" + length + - ", author=" + author + - '}'; + ", length=" + length ; + if (author!=null){ + s+= ", author=" + author.getUser().getUserId() + '}'; + }else{ + s += '}'; + } + return s; } } \ No newline at end of file diff --git a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java index c90d569..cb834fc 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java @@ -1,92 +1,91 @@ /** * */ package org.wikitolearn.wikirating.repository; import java.util.List; import java.util.Set; import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.annotation.QueryResult; import org.springframework.data.neo4j.repository.GraphRepository; import org.springframework.data.repository.query.Param; import org.wikitolearn.wikirating.model.graph.Revision; import org.wikitolearn.wikirating.model.graph.queryresult.RevisionResult; /** * @author aletundo * */ public interface RevisionRepository extends GraphRepository { /** * * @param langRevId * @return */ Revision findByLangRevId(String langRevId); - @Query("MATCH (r:Revision {langRevId:{0}}) WITH r MATCH p=(r)-[s*0..1]-() RETURN r as revision, nodes(p), relationships(p)") - RevisionResult getRev(String id); /** * * @param lang * @return */ Set findByLang(String lang); /** * * @param userId * @return */ Set findByUserId(int userId); /** * This query returns all the Revisions of a Page (CourseLevelThree). * The direction -> of the link is important to traverse * only the chain of Revisions of the page without reaching other nodes. * @param langPageId * @return */ @Query("MATCH (p:CourseLevelThree {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r") Set findAllRevisionOfPage(String langPageId); /** * This query returns all the Revisions of a Page ordere from the oldest * to the newest. * @param langPageId * @return */ @Query("MATCH (p:CourseLevelThree {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r ORDER BY r.revId") List findAllRevisionOfPageOrdered(String langPageId); /** * This query returns all the Revisions after the LAST_VALIDATED_REVISION. * The revisions are returned ordered by revId from the oldest to the newest. * @param langPageId * @return */ @Query("MATCH (page:Page {langPageId:{0}})-[:LAST_VALIDATED_REVISION]->(rev:Revision) " + - "WITH rev MATCH (rev)<-[:PREVIOUS_REVISION*]-(rev:Revision) " + - "WITH pr MATCH path=(pr)-[r*0..1]-() " + - "RETURN pr as revision , nodes(path), relationships(path) ORDER BY pr.revId") - List findAllRevisionNotValidated(String langPageId); + "WITH rev MATCH (prev:Revision)<-[pr:PREVIOUS_REVISION]-(rev)<-[:PREVIOUS_REVISION*]-(nextr:Revision) " + + "WITH rev, prev,pr, nextr " + + "MATCH p=(nextr)-[r*0..1]-() " + + "RETURN rev as revision,prev,pr, nodes(p) as nodes, relationships(p) as rels ") + RevisionResult getNotValidatedRevisionsChain(String langPageId); /** * */ @Query("MATCH (Page {langPageId:{id}})-[:LAST_VALIDATED_REVISION]->(rev:Revision) " + - "WITH rev MATCH (rev)-[a:AUTHOR]-(s:User) " + - "RETURN rev as revision, a, s as author ") + "WITH rev MATCH p=(rev)-[a*0..1]-() " + + "RETURN rev as revision, nodes(p), relationships(p) ") RevisionResult findLastValidatedRevision(@Param("id") String langPageId); /** * This query return the previous revision of a Revision identified * by langRevId. * @param langRevId langRevId of the requested revision * @return the previous revision */ @Query("MATCH (r:Revision {langRevId:{0}})-[:PREVIOUS_REVISION]->(a:Revision) RETURN a") Revision findPreviousRevision(String langRevId); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index ecd5178..c3aa33d 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,257 +1,243 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.*; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.model.graph.CourseLevelThree; import org.wikitolearn.wikirating.model.graph.Revision; import org.wikitolearn.wikirating.model.graph.queryresult.RevisionResult; import org.wikitolearn.wikirating.repository.CourseLevelThreeRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; import static java.lang.Math.exp; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private CourseLevelThreeRepository courseLevelThreeRepository; /** * Initialize the revisions for the first time querying the MediaWiki API. * This method adds the revisions and sets the FIRST_REVISION, * LAST_REVISION and PREVIOUS_REVISION relationships. * Only the revisions of CourseLevelThree pages are fetched. * @param lang the domain language * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initRevisions(String lang, String apiUrl) { List pages = courseLevelThreeRepository.findByLang(lang); for(CourseLevelThree page : pages){ List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageId()); // Set the first and the last revisions for the current page page.setFirstRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); // Set the last validated revision to the first one. page.setLastValidatedRevision(revisions.get(0)); // Create the PreviousRevision links ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevId()); rev.setLang(lang); if (it.previousIndex() != 0) { rev.setPreviousRevision(revisions.get(it.previousIndex() - 1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); courseLevelThreeRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); } return CompletableFuture.completedFuture(true); } @Async public CompletableFuture calculateChangeCoefficientAllRevisions(String lang, String apiUrl){ LOG.info("Calculating changeCoefficient for all the revisions..."); Set revisions = revisionRepository.findByLang(lang); for (Revision rev : revisions){ double changeCoefficient = calculateChangeCoefficient(apiUrl, rev); rev.setChangeCoefficient(changeCoefficient); } revisionRepository.save(revisions); LOG.info("ChangeCoefficient calculated for all revisions."); return CompletableFuture.completedFuture(true); } /** * Add a new Revision to the graph. This method DOESN'T link the * revision to a Page. * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang, userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } /** * Delete a revision given its langPageId * @param langPageId the langPageId of the revision */ public void deleteRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } revisionRepository.delete(revisions); } public Set getRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } return revisions; } /** * Get the revisions of a page ordered by revId from the oldest to the newest. * @param langPageId * @return * @throws RevisionNotFoundException */ public List getRevisionsOfPageOrdered(String langPageId) throws RevisionNotFoundException{ List revisions = revisionRepository.findAllRevisionOfPageOrdered(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } return revisions; } /** * Get all the revisions of a page from the LAST_VALIDATED_REVISION (included) to the * newest one ordered by revId. * @param langPageId * @return * @throws RevisionNotFoundException */ - public List getRevisionsFromLastValidated(String langPageId) throws RevisionNotFoundException{ - //List revisions = revisionRepository.findAllRevisionNotValidated(langPageId); - List revisions = new ArrayList<>(); - // We want also to add the latest validated revision to re-validate it. - // We put it at the beginning of the list because they are ordered from the oldest. - RevisionResult s = revisionRepository.getRev("es_196"); - RevisionResult r = revisionRepository.findLastValidatedRevision(langPageId); - //revisions.add(0, ); - /*if (revisions.size() == 0){ - LOG.error("Revisions of page {} not found", langPageId); - throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); - } - List result = new ArrayList<>(); - for (RevisionResult rev : revisions){ - //result.add(rev.revision); - }*/ - return result; + public Revision getRevisionsChainFromLastValidated(String langPageId) throws RevisionNotFoundException { + RevisionResult lastValidatedRevision = revisionRepository.getNotValidatedRevisionsChain(langPageId); + return lastValidatedRevision.revision; } /** * Get the requested revision * @param langRevId the langRevId of the revision * @return the revision * @throws RevisionNotFoundException */ public Revision getRevision(String langRevId) throws RevisionNotFoundException{ Revision revision = revisionRepository.findByLangRevId(langRevId); if(revision == null){ LOG.error("Revision {} not found", langRevId); throw new RevisionNotFoundException(); } return revision; } /** * Update the given revision * @param revision * @return the updated revision */ public Revision updateRevision(Revision revision){ revisionRepository.save(revision); return revision; } public void updateRevisions(Collection revisions){ revisionRepository.save(revisions); } /** * Calculate and set the changeCoefficient of a Revision. * This method also persist the Revision changes. * @param apiUrl * @param revision */ public void setChangeCoefficient(String apiUrl, Revision revision){ double cc = calculateChangeCoefficient(apiUrl, revision); revision.setChangeCoefficient(cc); revisionRepository.save(revision); } /** * Calculate the changeCoefficient for a given Revision querying mediawiki * and returns the number. It doesn't store it in the Revision. * @param revision * @return */ public double calculateChangeCoefficient(String apiUrl, Revision revision){ double previousLength = 0.0; double changeCoefficient = 0.0; // Get the previous Revision Revision previousRevision = revision.getPreviousRevision(); if (previousRevision == null){ previousLength = 1.0; changeCoefficient = 0.0; LOG.info("Change coefficient of revision {} (first-rev): {}", revision.getLangRevId(), changeCoefficient); } else{ double prevL = (double) previousRevision.getLength(); // Suppose the mean line length of 120 characters and that the length is in bytes. // We want a "lenght" in n° of lines if (prevL == 0){ previousLength = 1.0; }else { previousLength = (prevL < 120) ? 1 : prevL / 120; } // Query mediawiki for diff text String diffText = revisionMediaWikiService.getDiffPreviousRevision(apiUrl, previousRevision.getRevId(), revision.getRevId()); int addedLines = StringUtils.countMatches(diffText, "diff-addedline"); int deletedLines = StringUtils.countMatches(diffText, "diff-deletedline"); //int inlineChanges = StringUtils.countMatches(diffText, "diffchange-inline"); // Finally calculation of change Coefficient double t = ((1.2 * deletedLines + addedLines) ) / previousLength; changeCoefficient = 1 / exp(0.5 * t); LOG.info("Change coefficient of revision {} (+{}-{}/{}): {}", revision.getLangRevId(), addedLines, deletedLines, previousLength, changeCoefficient); } return changeCoefficient; } }