diff --git a/src/main/java/org/wikitolearn/wikirating/model/Page.java b/src/main/java/org/wikitolearn/wikirating/model/Page.java index 03936cc..a16c0df 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/Page.java +++ b/src/main/java/org/wikitolearn/wikirating/model/Page.java @@ -1,153 +1,154 @@ /** * */ package org.wikitolearn.wikirating.model; import org.neo4j.ogm.annotation.GraphId; import org.neo4j.ogm.annotation.Index; import org.neo4j.ogm.annotation.NodeEntity; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.neo4j.ogm.annotation.Relationship; /** * * @author aletundo, valsdav * */ @JsonIgnoreProperties(ignoreUnknown = true) @NodeEntity( label = "Page") public class Page { @GraphId private Long graphId; private int pageid; + @Index private String title; @Index private String lang; @Index(unique = true, primary=true) private String langPageId; private double pageRank; @Relationship(type = "LAST_REVISION", direction = Relationship.OUTGOING) private Revision lastRevision; @Relationship(type = "FIRST_REVISION", direction= Relationship.OUTGOING) private Revision fistRevision; /** * */ public Page() { } /** * @param title * @param pageid - * @param pageRank */ - public Page(String title, int pageid, double pageRank) { + public Page(String title, int pageid, String lang) { this.title = title; this.pageid = pageid; - this.pageRank = pageRank; + this.lang = lang; + this.langPageId = lang +"_"+ this.pageid; } /** * @return the title */ public String getTitle() { return title; } /** * @param title the title to set */ public void setTitle(String title) { this.title = title; } /** * @return the pageid */ public int getPageid() { return pageid; } /** * @param pageid the pageid to set */ public void setPageid(int pageid) { this.pageid = pageid; } /** * @return the pageRank */ public double getPageRank() { return pageRank; } /** * @param pageRank the pageRank to set */ public void setPageRank(double pageRank) { this.pageRank = pageRank; } /** * @return the lang */ public String getLang() { return lang; } /** * @param lang the lang to set */ public void setLang(String lang) { this.lang = lang; } /** * @return the langPageId */ public String getLangPageId() { return langPageId; } /** * @param langPageId the langPageId to set */ public void setLangPageId(String langPageId) { this.langPageId = langPageId; } /** * * @return */ public Revision getLastRevision() { return lastRevision; } /** * * @param lastRevision */ public void setLastRevision(Revision lastRevision) { this.lastRevision = lastRevision; } /** * * @return */ public Revision getFistRevision() { return fistRevision; } /** * * @param fistRevision */ public void setFistRevision(Revision fistRevision) { this.fistRevision = fistRevision; } /* (non-Javadoc) * @see java.lang.Object#toString() */ @Override public String toString() { return "Page [title=" + title + ", pageid=" + pageid + ", pageRank=" + pageRank + "]"; } } diff --git a/src/main/java/org/wikitolearn/wikirating/model/Revision.java b/src/main/java/org/wikitolearn/wikirating/model/Revision.java index a913cb2..279bec8 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/Revision.java +++ b/src/main/java/org/wikitolearn/wikirating/model/Revision.java @@ -1,372 +1,356 @@ package org.wikitolearn.wikirating.model; import org.neo4j.ogm.annotation.GraphId; import org.neo4j.ogm.annotation.Index; import org.neo4j.ogm.annotation.NodeEntity; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.neo4j.ogm.annotation.Relationship; import org.neo4j.ogm.annotation.typeconversion.DateLong; import java.util.Date; import java.util.Set; /** * This class handles the data of the Revision of a page. * Created by valsdav on 14/03/17. */ @JsonIgnoreProperties(ignoreUnknown = true) @NodeEntity( label = "Revision" ) public class Revision { @GraphId private Long graphId; private int revid; private String lang; private int userid; private int parentid; @DateLong private Date timestamp; private long length; private double changeCoefficient; private double currentMeanVote; private double currentVotesReliability; private double currentNormalisesVotesReliability; private double totalMeanVote; private double totalVotesReliability; private double totalNormalisesVotesReliability; private boolean validated; @Index(unique = true, primary=true) private String langRevId; @Relationship(type="PREVIOUS_REVISION", direction = Relationship.OUTGOING) private Revision previousRevision; @Relationship(type="VOTE", direction = Relationship.INCOMING) private Set votes; @Relationship( type="AUTHOR", direction = Relationship.INCOMING) private User author; /** * */ public Revision() {} /** * @param revid * @param userid * @param parentid * @param length - * @param changeCoefficient - * @param currentMeanVote - * @param currentVotesReliability - * @param currentNormalisesVotesReliability - * @param totalMeanVote - * @param totalVotesReliability - * @param totalNormalisesVotesReliability - * @param validated * @param lang * @param timestamp */ - public Revision(int revid, int userid, int parentid, long length, double changeCoefficient, double currentMeanVote, - double currentVotesReliability, double currentNormalisesVotesReliability, double totalMeanVote, - double totalVotesReliability, double totalNormalisesVotesReliability, boolean validated, String lang, Date timestamp) { + public Revision(int revid, String lang, int userid, int parentid, long length, Date timestamp) { this.revid = revid; + this.langRevId = lang +"_"+revid; this.userid = userid; this.parentid = parentid; this.length = length; - this.changeCoefficient = changeCoefficient; - this.currentMeanVote = currentMeanVote; - this.currentVotesReliability = currentVotesReliability; - this.currentNormalisesVotesReliability = currentNormalisesVotesReliability; - this.totalMeanVote = totalMeanVote; - this.totalVotesReliability = totalVotesReliability; - this.totalNormalisesVotesReliability = totalNormalisesVotesReliability; - this.validated = validated; this.lang = lang; this.timestamp = timestamp; + this.validated = false; } /** * @return the revid */ public int getRevid() { return revid; } /** * @param revid the revid to set */ public void setRevid(int revid) { this.revid = revid; } /** * @return the userid */ public int getUserid() { return userid; } /** * @param userid the userid to set */ public void setUserid(int userid) { this.userid = userid; } /** * @return the parentid */ public int getParentid() { return parentid; } /** * @param parentid the parentid to set */ public void setParentid(int parentid) { this.parentid = parentid; } /** * @return the length */ public long getLength() { return length; } /** * @param length the length to set */ public void setLength(long length) { this.length = length; } /** * @return the changeCoefficient */ public double getChangeCoefficient() { return changeCoefficient; } /** * @param changeCoefficient the changeCoefficient to set */ public void setChangeCoefficient(double changeCoefficient) { this.changeCoefficient = changeCoefficient; } /** * @return the currentMeanVote */ public double getCurrentMeanVote() { return currentMeanVote; } /** * @param currentMeanVote the currentMeanVote to set */ public void setCurrentMeanVote(double currentMeanVote) { this.currentMeanVote = currentMeanVote; } /** * @return the currentVotesReliability */ public double getCurrentVotesReliability() { return currentVotesReliability; } /** * @param currentVotesReliability the currentVotesReliability to set */ public void setCurrentVotesReliability(double currentVotesReliability) { this.currentVotesReliability = currentVotesReliability; } /** * @return the currentNormalisesVotesReliability */ public double getCurrentNormalisesVotesReliability() { return currentNormalisesVotesReliability; } /** * @param currentNormalisesVotesReliability the currentNormalisesVotesReliability to set */ public void setCurrentNormalisesVotesReliability(double currentNormalisesVotesReliability) { this.currentNormalisesVotesReliability = currentNormalisesVotesReliability; } /** * @return the totalMeanVote */ public double getTotalMeanVote() { return totalMeanVote; } /** * @param totalMeanVote the totalMeanVote to set */ public void setTotalMeanVote(double totalMeanVote) { this.totalMeanVote = totalMeanVote; } /** * @return the totalVotesReliability */ public double getTotalVotesReliability() { return totalVotesReliability; } /** * @param totalVotesReliability the totalVotesReliability to set */ public void setTotalVotesReliability(double totalVotesReliability) { this.totalVotesReliability = totalVotesReliability; } /** * @return the totalNormalisesVotesReliability */ public double getTotalNormalisesVotesReliability() { return totalNormalisesVotesReliability; } /** * @param totalNormalisesVotesReliability the totalNormalisesVotesReliability to set */ public void setTotalNormalisesVotesReliability(double totalNormalisesVotesReliability) { this.totalNormalisesVotesReliability = totalNormalisesVotesReliability; } /** * @return the validated */ public boolean isValidated() { return validated; } /** * @param validated the validated to set */ public void setValidated(boolean validated) { this.validated = validated; } /** * @return the lang */ public String getLang() { return lang; } /** * @param lang the lang to set */ public void setLang(String lang) { this.lang = lang; } /** * @return the langRevId */ public String getLangRevId() { return langRevId; } /** * @param langRevId the langRevId to set */ public void setLangRevId(String langRevId) { this.langRevId = langRevId; } /** * @return the graphId */ public Long getGraphId() { return graphId; } /** * @param graphId the graphId to set */ public void setGraphId(Long graphId) { this.graphId = graphId; } /** * @return the timestamp */ public Date getTimestamp() { return timestamp; } /** * @param timestamp the timestamp to set */ public void setTimestamp(Date timestamp) { this.timestamp = timestamp; } /** * @return the previousRevision */ public Revision getPreviousRevision() { return previousRevision; } /** * @param previousRevision the previousRevision to set */ public void setPreviousRevision(Revision previousRevision) { this.previousRevision = previousRevision; } /** * @return the votes */ public Set getVotes() { return votes; } /** * @param votes the votes to set */ public void setVotes(Set votes) { this.votes = votes; } /** * @return the author */ public User getAuthor() { return author; } /** * @param author the author to set */ public void setAuthor(User author) { this.author = author; } /* (non-Javadoc) * @see java.lang.Object#toString() */ @Override public String toString() { return "Revision{" + "graphId=" + graphId + ", revid=" + revid + ", lang='" + lang + '\'' + ", userid=" + userid + ", parentid=" + parentid + ", timestamp=" + timestamp + ", length=" + length + ", changeCoefficient=" + changeCoefficient + ", validated=" + validated + ", langRevId='" + langRevId + '\'' + ", author=" + author + '}'; } } \ No newline at end of file diff --git a/src/main/java/org/wikitolearn/wikirating/model/UpdateEvent.java b/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java similarity index 77% rename from src/main/java/org/wikitolearn/wikirating/model/UpdateEvent.java rename to src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java index 611638e..65f4a71 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/UpdateEvent.java +++ b/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java @@ -1,112 +1,109 @@ package org.wikitolearn.wikirating.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.wikitolearn.wikirating.util.enums.UpdateType; import java.util.Date; import java.util.Map; /** * This class represents a RecentChange entry during the fetch Process. * It stores the information fetched from the API before the update of the DB. * Created by valsdav on 30/03/17. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class UpdateEvent { +public class UpdateInfo { private UpdateType type; + private String title; private int pageid; private int revid; private int old_revid; + private String user; private int userid; private int oldlen; private int newlen; - private UpdateType logtype; - private Map logparams; private Date timestamp; - public UpdateEvent(){} + public UpdateInfo(){} public UpdateType getType() { - if (this.type == UpdateType.LOG){ - return this.logtype; - } return type; } public void setType(UpdateType type) { this.type = type; } public int getPageid() { return pageid; } public void setPageid(int pageid) { this.pageid = pageid; } public int getRevid() { return revid; } public void setRevid(int revid) { this.revid = revid; } public int getOld_revid() { return old_revid; } public void setOld_revid(int old_revid) { this.old_revid = old_revid; } public int getUserid() { return userid; } public void setUserid(int userid) { this.userid = userid; } public int getOldlen() { return oldlen; } public void setOldlen(int oldlen) { this.oldlen = oldlen; } public int getNewlen() { return newlen; } public void setNewlen(int newlen) { this.newlen = newlen; } public Date getTimestamp() { return timestamp; } public void setTimestamp(Date timestamp) { this.timestamp = timestamp; } - public UpdateType getLogtype() { - return logtype; + public String getTitle() { + return title; } - public void setLogtype(UpdateType logtype) { - this.logtype = logtype; + public void setTitle(String title) { + this.title = title; } - public Map getLogparams() { - return logparams; + public String getUser() { + return user; } - public void setLogparams(Map logparams) { - this.logparams = logparams; + public void setUser(String user) { + this.user = user; } } diff --git a/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java index 8631c87..1736bc5 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java @@ -1,27 +1,28 @@ /** * */ package org.wikitolearn.wikirating.repository; import org.springframework.data.neo4j.repository.GraphRepository; import org.wikitolearn.wikirating.model.Page; /** * @author aletundo * */ public interface PageRepository extends GraphRepository { /** * * @param title + * @param lang * @return */ - Page findByTitle(String title); - + Page findByTitleAndLang(String title, String lang); + /** * * @param langPageId * @return */ Page findByLangPageId(String langPageId); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/PageService.java b/src/main/java/org/wikitolearn/wikirating/service/PageService.java index e6c91f1..bd0167a 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/PageService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/PageService.java @@ -1,50 +1,103 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.List; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.model.Page; +import org.wikitolearn.wikirating.model.Revision; +import org.wikitolearn.wikirating.model.UpdateInfo; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.service.mediawiki.PageMediaWikiService; /** * * @author aletundo, valsdav * */ @Service public class PageService { private static final Logger LOG = LoggerFactory.getLogger(PageService.class); - @Autowired - private PageMediaWikiService pageMediaWikiService; - @Autowired - private PageRepository pageRepository; + + @Autowired private PageMediaWikiService pageMediaWikiService; + @Autowired RevisionService revisionService; + @Autowired private PageRepository pageRepository; /** * This methods inserts all the pages inside the DB querying the MediaWiki API. * @param lang String * @param apiUrl String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture addAllPages( String lang, String apiUrl ){ List pages = pageMediaWikiService.getAll(apiUrl); pages.forEach(page -> { page.setLang(lang); page.setLangPageId(lang + "_" +page.getPageid()); }); pageRepository.save(pages); LOG.info("Inserted all {} pages", lang); return CompletableFuture.completedFuture(true); } + + /** + * This method creates a new Page. It requires the firstR evision of the Page in order + * to create the LAST_REVISION and FIRST_REVISION relationships. + * @param pageid + * @param title + * @param lang + * @param firstRevision + * @return + */ + public Page addPage(int pageid, String title, String lang, Revision firstRevision){ + Page page = new Page(title, pageid, lang); + //creating the relations with the first revision. + page.setFistRevision(firstRevision); + page.setLastRevision(firstRevision); + pageRepository.save(page); + return page; + } + + /** + * This method adds a new Revision to a page. It links the Page to the new revision via + * LAST_REVISION link. Moreover it create the PREVIOUS_REVISION link. + * @param langPageId + * @param rev + * @return + */ + public Page addRevisionToPage(String langPageId, Revision rev){ + Page page = pageRepository.findByLangPageId(langPageId); + //adding PREVIOUS_REVISION relationship + rev.setPreviousRevision(page.getLastRevision()); + page.setLastRevision(rev); + //the changes on the revision will be automatically persisted + pageRepository.save(page); + return page; + } + + /** + * This method changes only the title of a given Page. + * @param oldTitle + * @param newTitle + * @param lang + * @return + */ + public Page movePage(String oldTitle, String newTitle, String lang){ + Page page = pageRepository.findByTitleAndLang(oldTitle, lang); + page.setTitle(newTitle); + pageRepository.save(page); + return page; + } + + } diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index c5d22da..535c5a9 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,73 +1,87 @@ /** * */ package org.wikitolearn.wikirating.service; -import java.util.HashMap; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; +import java.util.*; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.model.Page; import org.wikitolearn.wikirating.model.Revision; +import org.wikitolearn.wikirating.model.UpdateInfo; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private PageRepository pageRepository; /** * This method inserts all the revisions for every page, creating the * connections between them and between the users that have written them. * * @param lang * String * @param apiUrl * String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture addAllRevisions(String lang, String apiUrl) { Iterable pages = pageRepository.findAll(); pages.forEach(page -> { List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageid()); // Set the first and the last revisions for the current page page.setFistRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevid()); rev.setLang(lang); if (it.previousIndex() != 0){ rev.setPreviousRevision(revisions.get(it.previousIndex()-1)); } } //saving all the revisions node revisionRepository.save(revisions); pageRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); }); return CompletableFuture.completedFuture(true); } + /** + * This method adds a new Revision to the DB. + * @param revid + * @param lang + * @param userid + * @param parentid + * @param length + * @param timestamp + * @return + */ + public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ + Revision rev = new Revision(revid, lang,userid, parentid, length, timestamp); + revisionRepository.save(rev); + return rev; + } + } diff --git a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java index 4f7332f..d6c0433 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java @@ -1,52 +1,83 @@ package org.wikitolearn.wikirating.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.model.Process; -import org.wikitolearn.wikirating.model.UpdateEvent; -import org.wikitolearn.wikirating.service.mediawiki.RecentChangesMediaWikiService; +import org.wikitolearn.wikirating.model.Revision; +import org.wikitolearn.wikirating.model.UpdateInfo; +import org.wikitolearn.wikirating.service.mediawiki.UpdateMediaWikiService; import org.wikitolearn.wikirating.util.enums.ProcessType; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.concurrent.CompletableFuture; /** * Created by valsdav on 29/03/17. */ @Service public class UpdateService { private static final Logger LOG = LoggerFactory.getLogger(UpdateService.class); @Autowired private PageService pageService; @Autowired private UserService userService; @Autowired private RevisionService revisionService; @Autowired private ProcessService processService; @Autowired private MetadataService metadataService; - @Autowired private RecentChangesMediaWikiService recentChangesMediaWikiService; + @Autowired private UpdateMediaWikiService updateMediaWikiService; @Value("#{'${mediawiki.langs}'.split(',')}") private List langs; @Value("${mediawiki.protocol}") private String protocol; @Value("${mediawiki.api.url}") private String apiUrl; + @Value("${mediawiki.namespace}") + private String namespace; private boolean updateData(){ //getting the begin timestamp of the latest FETCH Process before opening a new process Date beginOfLastFetch = processService.getLastProcessStartDateByType(ProcessType.FETCH); //opening a new FETCH process Process currentFetchProcess = processService.createNewProcess(ProcessType.FETCH); //First of all we get the RecentChangeEvents from Mediawiki API. - String url = protocol + langs.get(0) + "." + apiUrl; - List updates = recentChangesMediaWikiService.getRecentChangesBetweenDates(apiUrl,"2800", - beginOfLastFetch, currentFetchProcess.getStartOfProcess()); + for(String lang : langs) { + String url = protocol + lang + "." + apiUrl; + //fetching upda + List updates = updateMediaWikiService.getPagesUpdateInfo(url, namespace, + beginOfLastFetch, currentFetchProcess.getStartOfProcess()); + + updates.forEach(update -> { + switch (update.getType()) { + case NEW: + //create the new revision + Revision newRev = revisionService.addRevision(update.getRevid(), lang, + update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); + //then create a new Page and link it with the revision + pageService.addPage(update.getPageid(), update.getTitle(), lang, newRev); + break; + case EDIT: + //create a new revision + Revision updateRev = revisionService.addRevision(update.getRevid(), lang, + update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); + //then add it to the page + pageService.addRevisionToPage(lang + "_" + update.getPageid(), updateRev); + break; + case MOVE: + break; + case DELETE: + break; + } + }); + + } return true; } - } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RecentChangesMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RecentChangesMediaWikiService.java deleted file mode 100644 index a34260d..0000000 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RecentChangesMediaWikiService.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.wikitolearn.wikirating.service.mediawiki; - -import com.fasterxml.jackson.core.type.TypeReference; -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.springframework.stereotype.Service; -import org.wikidata.wdtk.wikibaseapi.ApiConnection; -import org.wikitolearn.wikirating.model.UpdateEvent; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; - -/** - * Created by valsdav on 29/03/17. - */ -@Service -public class RecentChangesMediaWikiService extends MediaWikiService{ - - public List getRecentChangesBetweenDates(String apiUrl, String namespace, Date begin, Date end){ - ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); - InputStream response; - boolean moreRc = true; - JSONArray rcJson = new JSONArray(); - List toBeConcat = new ArrayList<>(); - List rcs = new ArrayList<>(); - Map parameters = mediaWikiApiUtils.getRecentChangesParam(namespace, begin,end); - try { - while(moreRc){ - response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); - JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); - - toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("recentchanges")); - - if(responseJson.has("continue")){ - String continueFrom = responseJson.getJSONObject("continue").getString("rccontinue"); - parameters.put("rccontinue", continueFrom); - }else{ - moreRc = false; - rcJson = concatArrays(toBeConcat); - } - } - rcs = mapper.readValue(rcJson.toString(), new TypeReference>(){}); - return rcs; - } catch (JSONException e){ - LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); - } catch(IOException e){ - LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); - } - return rcs; - - } - - @Override - public List getAll(String apiUrl) { - return null; - } -} diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java new file mode 100644 index 0000000..2a09ac5 --- /dev/null +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java @@ -0,0 +1,114 @@ +package org.wikitolearn.wikirating.service.mediawiki; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.stereotype.Service; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikitolearn.wikirating.model.UpdateInfo; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Created by valsdav on 29/03/17. + */ +@Service +public class UpdateMediaWikiService extends MediaWikiService{ + + public List getPagesUpdateInfo(String apiUrl, String namespace, Date start, Date end){ + List editsAndNewPages = getRecentChangesBetweenDates(apiUrl, namespace, start, end); + List deletedPages = getLogEventsBetweenDates(apiUrl, "delete", start, end); + List movedPages = getLogEventsBetweenDates(apiUrl, "move", start, end); + List allUpdates = Stream.of(editsAndNewPages,deletedPages,movedPages) + .flatMap(List::stream).collect(Collectors.toList()); + return allUpdates; + } + + public List getNewUsers(String apiUrl, Date start, Date end){ + return getLogEventsBetweenDates(apiUrl, "newusers", start, end); + } + + + public List getRecentChangesBetweenDates( String apiUrl, String namespace, Date start, Date end){ + ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); + InputStream response; + boolean moreRc = true; + JSONArray rcJson = new JSONArray(); + List toBeConcat = new ArrayList<>(); + List rcs = new ArrayList<>(); + Map parameters = mediaWikiApiUtils.getRecentChangesParam(namespace, start,end); + try { + while(moreRc){ + response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); + JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); + + toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("recentchanges")); + + if(responseJson.has("continue")){ + String continueFrom = responseJson.getJSONObject("continue").getString("rccontinue"); + parameters.put("rccontinue", continueFrom); + }else{ + moreRc = false; + rcJson = concatArrays(toBeConcat); + } + } + rcs = mapper.readValue(rcJson.toString(), new TypeReference>(){}); + return rcs; + } catch (JSONException e){ + LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); + } catch(IOException e){ + LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); + } + return rcs; + } + + + public List getLogEventsBetweenDates( String apiUrl, String logtype, Date start, Date end){ + ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); + InputStream response; + JSONArray leJson = new JSONArray(); + List toBeConcat = new ArrayList<>(); + List levents = new ArrayList<>(); + + try { + Map parameters = mediaWikiApiUtils.getLogEventsParam(logtype, start,end); + boolean moreLe = true; + + while(moreLe){ + response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); + JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); + + toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("recentchanges")); + + if(responseJson.has("continue")){ + String continueFrom = responseJson.getJSONObject("continue").getString("lecontinue"); + parameters.put("lecontinue", continueFrom); + }else{ + moreLe = false; + leJson = concatArrays(toBeConcat); + } + } + levents = mapper.readValue(leJson.toString(), new TypeReference>(){}); + return levents; + } catch (JSONException e){ + LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); + } catch(IOException e){ + LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); + } + return levents; + } + + + @Override + public List getAll(String apiUrl) { + return null; + } +} diff --git a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java index 9ad7675..1623eb2 100644 --- a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java +++ b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java @@ -1,198 +1,220 @@ /** * */ package org.wikitolearn.wikirating.util; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.HashMap; import java.util.Map; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikidata.wdtk.wikibaseapi.LoginFailedException; import org.wikitolearn.wikirating.exception.GenericException; /** * @author aletundo, valsdav * */ @Component public class MediaWikiApiUtils { private static final Logger LOG = LoggerFactory.getLogger(MediaWikiApiUtils.class); @Value("${mediawiki.api.user}") private String apiUser; @Value("${mediawiki.api.password}") private String apiPassword; /** * This method creates a MediaWiki Connection object * @param apiUrl the url of the API * @return ApiConnection the API connection object */ public ApiConnection getApiConnection(String apiUrl) { ApiConnection connection = new ApiConnection(apiUrl); if(connection.isLoggedIn()){ return connection; }else{ try { connection.login(apiUser, apiPassword); } catch (LoginFailedException e) { LOG.error("MediaWiki login failed. {}", e.getMessage()); // TODO change exception throw new GenericException(e.getMessage()); } return connection; } } /** * This method constructs the Map of parameters to attach with the MediaWiki Query to fetch all the pages * in the specified namespace * @param namespace The namespace whose pages are requested * @return Map having parameters */ public Map getListAllPagesParamsMap(String namespace) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allpages"); queryParameterMap.put("aplimit", "max"); queryParameterMap.put("apnamespace", namespace); queryParameterMap.put("apfilterredir", "nonredirects"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to fetch all the revisions * of the given page * @param pid The PageID of the page for which revisions are requested * @return Map having parameters */ public Map getRevisionParam(int pid) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("prop", "revisions"); queryParameterMap.put("pageids", Integer.toString(pid)); queryParameterMap.put("rvprop", "userid|ids|timestamp|flags|size"); queryParameterMap.put("rvlimit", "max"); queryParameterMap.put("rvdir", "newer"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to get * all the users. * @return Map having parameters */ public Map getUserParam() { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allusers"); queryParameterMap.put("aulimit", "max"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** - * This method constructs the MAP of parameters to attach iwth the Mediawiki Query to get + * This method constructs the MAP of parameters to attach to the Mediawiki Query to get * all the recent changes in one namespace between two dates. * @param namespace namespace to user * @param begin start of the recentchanges * @param end end of the changes * @return the map with the parameters */ public Map getRecentChangesParam(String namespace, Date begin, Date end){ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); Map queryParameterMap = new HashMap<>(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "recentchanges"); queryParameterMap.put("rclimit", "max"); queryParameterMap.put("format", "json"); queryParameterMap.put("rcnamespace", namespace); queryParameterMap.put("rcshow", "!bot|!redirect"); - queryParameterMap.put("rcprop", "userid|timestamp|ids|sizes|flags|loginfo"); - queryParameterMap.put("rctype", "new|edit|log"); + queryParameterMap.put("rcprop", "title|userid|timestamp|ids|sizes|flags"); + queryParameterMap.put("rctype", "new|edit"); queryParameterMap.put("rcstart", dateFormat.format(begin)); queryParameterMap.put("rcend", dateFormat.format(end)); queryParameterMap.put("rcdir", "newer"); return queryParameterMap; } + + /** + * This method constructs the MAP of parameters to attach to the Mediawiki Query to get + * all the log entries between two dates. We need the logs about moved pages, new users, deleted pages. + * @param logtype Type of log to fetch: newusers|delete|move + * @param begin start of the recentchanges + * @param end end of the changes + * @return the map with the parameters + */ + public Map getLogEventsParam(String logtype, Date begin, Date end){ + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + Map queryParameterMap = new HashMap<>(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("list", "logevents"); + queryParameterMap.put("lelimit", "max"); + queryParameterMap.put("format", "json"); + queryParameterMap.put("leprop", "ids|title|user|userid|timestamp"); + queryParameterMap.put("lestart", dateFormat.format(begin)); + queryParameterMap.put("leend", dateFormat.format(end)); + queryParameterMap.put("ledir", "newer"); + return queryParameterMap; + } /** * This method converts an InputStream object to String * @param inputStream InputStream object to be converted * @return String result the converted stream into a string */ public JSONObject streamToJson(InputStream inputStream) { String result = ""; BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); } catch (UnsupportedEncodingException e1) { LOG.error("Unsupported enconding. {}", e1.getMessage()); } StringBuilder builder = new StringBuilder(); String line; try { if(reader!=null){ while ((line = reader.readLine()) != null) { builder.append(line); } } result = builder.toString(); inputStream.close(); } catch (IOException e) { LOG.error("An error occurs while reading the inputStream. {}", e.getMessage()); } JSONObject jsonResponse = null; try { jsonResponse = new JSONObject(result); } catch (JSONException e) { LOG.error("An error occurs while converting string to JSONObject. {}", e.getMessage()); } return jsonResponse; } /** * This method sends a request to MediaWiki API and then gets back an InputStream * @param connection ApiConnection The ApiConnection object * @param requestMethod RequestMethod The request method (ex: GET, POST, ...) * @param queryParametersMap Map The HashMap having all the query parameters * @return response InputStream The result data */ public InputStream sendRequest(ApiConnection connection, String requestMethod, Map queryParametersMap) { InputStream response = null; try { response = connection.sendRequest(requestMethod, queryParametersMap); } catch (IOException e) { LOG.error("Failed to send a request to MediaWiki API. {}", e.getMessage()); } return response; } } diff --git a/src/main/java/org/wikitolearn/wikirating/util/enums/UpdateType.java b/src/main/java/org/wikitolearn/wikirating/util/enums/UpdateType.java index c84b725..64d669c 100644 --- a/src/main/java/org/wikitolearn/wikirating/util/enums/UpdateType.java +++ b/src/main/java/org/wikitolearn/wikirating/util/enums/UpdateType.java @@ -1,8 +1,8 @@ package org.wikitolearn.wikirating.util.enums; /** * Created by valsdav on 30/03/17. */ public enum UpdateType { - NEW, EDIT, MOVE, DELETE, LOG; + NEW, EDIT, MOVE, DELETE, NEWUSERS; }