diff --git a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java index dca3302..b4f5615 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/RevisionRepository.java @@ -1,49 +1,58 @@ /** * */ package org.wikitolearn.wikirating.repository; +import java.util.List; import java.util.Set; import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.repository.GraphRepository; import org.wikitolearn.wikirating.model.Revision; /** * @author aletundo * */ public interface RevisionRepository extends GraphRepository { /** * * @param langRevId * @return */ Revision findByLangRevId(String langRevId); - + + /** + * + * @param lang + * @return + */ + @Query("MATCH (r:Revision {lang:{0}}) RETURN r") + Set findAllByLang(String lang); + /** * * @param userId * @return */ Set findByUserId(int userId); /** * This query returns all the Revisions of a Page. * The direction -> of the link is important to traverse * only the chain of Revisions of the page without reaching other nodes. * @param langPageId * @return */ @Query("MATCH (p:Page {langPageId:{0}})-[:LAST_REVISION|PREVIOUS_REVISION*]->(r:Revision) RETURN r") Set findAllRevisionOfPage(String langPageId); /** * This query return the previous revision of a Revision identified * by langRevId. * @param langRevId langRevId of the requested revision * @return the previous revision */ @Query("MATCH (r:Revision {langRevId:{0}})-[:PREVIOUS_REVISION]->(a:Revision) RETURN a") Revision findPreviousRevision(String langRevId); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java b/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java index 3ab1317..46a9e21 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java @@ -1,239 +1,255 @@ package org.wikitolearn.wikirating.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.*; import org.wikitolearn.wikirating.model.Process; import org.wikitolearn.wikirating.util.enums.ProcessStatus; import org.wikitolearn.wikirating.util.enums.ProcessType; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; /** * @author aletundo * @author valsdav */ @Service public class MaintenanceService { private static final Logger LOG = LoggerFactory.getLogger(MaintenanceService.class); @Autowired private UserService userService; @Autowired private PageService pageService; @Autowired private RevisionService revisionService; @Autowired private MetadataService metadataService; @Autowired private ProcessService processService; @Autowired private VoteService voteService; @Value("#{'${mediawiki.langs}'.split(',')}") private List langs; @Value("${mediawiki.protocol}") private String protocol; @Value("${mediawiki.api.url}") private String apiUrl; @Value("${mediawiki.namespace}") private String namespace; /** * Initialize the graph for the first time using parallel threads for each domain language * @return true if initialization succeed */ public boolean initializeGraph() { // Initialize Metadata service metadataService.initMetadata(); // Start a new Process Process initProcess = processService.addProcess(ProcessType.INIT); metadataService.addFirstProcess(initProcess); CompletableFuture initFuture = CompletableFuture + // Users and pages .allOf(buildUsersAndPagesFuturesList().toArray(new CompletableFuture[langs.size() + 1])) .thenCompose(result -> CompletableFuture + // Revisions .allOf(buildRevisionsFuturesList().toArray(new CompletableFuture[langs.size()]))) + .thenCompose(result -> CompletableFuture + // Change Coefficients + .allOf(buildChangeCoefficientFuturesList().toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> CompletableFuture + // CourseStructure .allOf(buildInitCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> userService.initAuthorship()); try { boolean result = initFuture.get(); // Save the result of the process if (result){ processService.closeCurrentProcess(ProcessStatus.DONE); }else{ processService.closeCurrentProcess(ProcessStatus.EXCEPTION); } return result; } catch (InterruptedException | ExecutionException e) { LOG.error("Something went wrong. {}", e.getMessage()); return false; } } /** * Entry point for the scheduled graph updated * @return true if the update succeed */ @Scheduled(cron = "${maintenance.update.cron}") public void updateGraph() { Process currentFetchProcess; Date startTimestampCurrentFetch, startTimestampLatestFetch; // Get start timestamp of the latest FETCH Process before opening a new process startTimestampLatestFetch = (processService.getLastProcessStartDateByType(ProcessType.FETCH) != null) ? processService.getLastProcessStartDateByType(ProcessType.FETCH) : processService.getLastProcessStartDateByType(ProcessType.INIT); // Create a new FETCH process try { currentFetchProcess = processService.addProcess(ProcessType.FETCH); metadataService.updateLatestProcess(); startTimestampCurrentFetch = currentFetchProcess.getStartOfProcess(); } catch (PreviousProcessOngoingException e){ LOG.error("Cannot start Update process because the previous process is still ONGOING." + "The update will be aborted."); return; } try { CompletableFuture updateFuture = CompletableFuture .allOf(userService.updateUsers(protocol + langs.get(0) + "." + apiUrl, startTimestampLatestFetch, startTimestampCurrentFetch)) .thenCompose(result -> CompletableFuture .allOf(buildUpdatePagesFuturesList(startTimestampLatestFetch, startTimestampCurrentFetch) .toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> CompletableFuture .allOf(buildUpdateCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> voteService.validateTemporaryVotes(startTimestampCurrentFetch)); boolean result = updateFuture.get(); // Save the result of the process, closing the current one if (result) { processService.closeCurrentProcess(ProcessStatus.DONE); } else { processService.closeCurrentProcess(ProcessStatus.ERROR); } } catch (TemporaryVoteValidationException | UpdateUsersException | UpdatePagesAndRevisionsException | InterruptedException | ExecutionException e) { processService.closeCurrentProcess(ProcessStatus.EXCEPTION); LOG.error("An error occurred during a scheduled graph update procedure"); throw new UpdateGraphException(); } } /** * Build a list of CompletableFuture. * * @return a list of CompletableFuture */ private List> buildInitCourseStructureFuturesList() { List> parallelInitCourseStructureFutures = new ArrayList<>(); // Add course structure for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelInitCourseStructureFutures.add(pageService.initCourseStructure(lang, url)); } return parallelInitCourseStructureFutures; } /** * Build a list of CompletableFuture. * * @return a list of CompletableFuture */ private List> buildUpdateCourseStructureFuturesList() { List> parallelUpdateCourseStructureFutures = new ArrayList<>(); // Add course structure for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelUpdateCourseStructureFutures.add(pageService.updateCourseStructure(lang, url)); } return parallelUpdateCourseStructureFutures; } /** * * @param start * @param end * @return */ private List> buildUpdatePagesFuturesList(Date start, Date end) { List> futures = new ArrayList<>(); // Add update pages for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; futures.add(pageService.updatePages(lang, url, start, end)); } return futures; } /** * Build a list of CompletableFuture. The elements are the fetches of pages' * revisions from each domain language. * * @return a list of CompletableFuture */ private List> buildRevisionsFuturesList() { List> parallelRevisionsFutures = new ArrayList<>(); // Add revisions fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelRevisionsFutures.add(revisionService.initRevisions(lang, url)); } return parallelRevisionsFutures; } + private List> buildChangeCoefficientFuturesList(){ + List> parallelCCFutures = new ArrayList<>(); + // Add revisions fetch for each domain language + for (String lang : langs) { + String url = protocol + lang + "." + apiUrl; + parallelCCFutures.add(revisionService.calculateChangeCoefficientAllRevisions(lang, url)); + } + return parallelCCFutures; + } + /** * Build a list of CompletableFuture. The first one is the fetch of the * users from the first domain in mediawiki.langs list. The rest of the * elements are the fetches of the pages for each language. This * implementation assumes that the users are shared among domains. * * @return a list of CompletableFuture */ private List> buildUsersAndPagesFuturesList() { List> usersAndPagesInsertion = new ArrayList<>(); // Add users fetch as fist operation usersAndPagesInsertion.add(userService.initUsers(protocol + langs.get(0) + "." + apiUrl)); // Add pages fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; usersAndPagesInsertion.add(pageService.initPages(lang, url)); } return usersAndPagesInsertion; } /*@SuppressWarnings("unchecked") private List> buildFuturesList(Object obj, String methodPrefix) { List> futures = new ArrayList<>(); for (String lang : langs) { String url = protocol + lang + "." + apiUrl; Method[] methods = obj.getClass().getMethods(); for (int i = 0; i < methods.length; i++) { try { if (methods[i].getName().startsWith(methodPrefix) && methods[i].getParameterCount() == 1) { futures.add((CompletableFuture) methods[i].invoke(obj, url)); } else if (methods[i].getName().startsWith(methodPrefix) && methods[i].getParameterCount() == 2) { futures.add((CompletableFuture) methods[i].invoke(obj, lang, url)); } } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } return futures; }*/ } diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index 6f8a925..8e56859 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,170 +1,197 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.List; import java.util.ListIterator; import java.util.Set; import java.util.concurrent.CompletableFuture; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.model.Page; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; import static java.lang.Math.exp; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private PageRepository pageRepository; /** * Initialize the revisions for the first time querying the MediaWiki API. * This method adds the revisions and sets the FIRST_REVISION, * LAST_REVISION and PREVIOUS_REVISION relationships. * @param lang the domain language * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initRevisions(String lang, String apiUrl) { List pages = pageRepository.findAllByLang(lang); for(Page page : pages){ List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageId()); // Set the first and the last revisions for the current page page.setFistRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevId()); rev.setLang(lang); - if (it.previousIndex() != 0){ - rev.setPreviousRevision(revisions.get(it.previousIndex()-1)); + if (it.previousIndex() != 0) { + rev.setPreviousRevision(revisions.get(it.previousIndex() - 1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); pageRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); } return CompletableFuture.completedFuture(true); } + @Async + public CompletableFuture calculateChangeCoefficientAllRevisions(String lang, String apiUrl){ + LOG.info("Calculating changeCoefficient for all the revisions..."); + Set revisions = revisionRepository.findAllByLang(lang); + for (Revision rev : revisions){ + double changeCoefficient = calculateChangeCoefficient(apiUrl, rev); + rev.setChangeCoefficient(changeCoefficient); + } + revisionRepository.save(revisions); + LOG.info("ChangeCoefficient calculated for all revisions."); + return CompletableFuture.completedFuture(true); + } + /** * Add a new Revision to the graph * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang, userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } /** * Delete a revision given its langPageId * @param langPageId the langPageId of the revision */ public void deleteRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } revisionRepository.delete(revisions); } public Set getRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } return revisions; } /** * Get the requested revision * @param langRevId the langRevId of the revision * @return the revision * @throws RevisionNotFoundException */ public Revision getRevision(String langRevId) throws RevisionNotFoundException{ Revision revision = revisionRepository.findByLangRevId(langRevId); if(revision == null){ LOG.error("Revision {} not found", langRevId); throw new RevisionNotFoundException(); } return revision; } /** * Update the given revision * @param revision * @return the updated revision */ public Revision updateRevision(Revision revision){ revisionRepository.save(revision); return revision; } - - /** - * - * @param revisionId - * @param pageId + + /** + * Calculate the changeCoefficient for a given Revision querying mediawiki + * and returns the number. It doesn't store it in the Revision. + * @param revision * @return */ - public double calculateChangeCofficient(String apiUrl, int revisionId, int pageId, String langRevId){ - String diffText = revisionMediaWikiService.getDiffPreviousRevision(apiUrl, revisionId, pageId); - - int addedLines = StringUtils.countMatches(diffText, "diff-addedline"); - int deletedLines = StringUtils.countMatches(diffText, "diff-deletedline"); - int inlineChanges = StringUtils.countMatches(diffText, "diffchange-inline"); - - // Get the previous Revision for the previous size parameter - long previousLength = 0; - Revision previousRevision = revisionRepository.findPreviousRevision(langRevId); + public double calculateChangeCoefficient(String apiUrl, Revision revision){ + double previousLength = 0; + double changeCoefficient = 0; + // Get the previous Revision + Revision previousRevision = revision.getPreviousRevision(); if (previousRevision == null){ previousLength = 1; + changeCoefficient = 0; + LOG.info("Change coefficient of revision {} (first-rev): {}", revision.getLangRevId(), changeCoefficient); } else{ + double prevL = (double) previousRevision.getLength(); // Suppose the mean line length of 120 characters and that the length is in bytes. // We want a "lenght" in n° of lines - previousLength = previousRevision.getLength() / 240; - } + if (prevL == 0){ + previousLength = 1; + }else { + previousLength = (prevL < 240) ? 1 : prevL / 240; + } + + // Query mediawiki for diff text + String diffText = revisionMediaWikiService.getDiffPreviousRevision(apiUrl, + previousRevision.getRevId(), revision.getRevId()); - System.out.println("add: " + addedLines + "\n del:" + deletedLines + "\n inline: " + inlineChanges); + int addedLines = StringUtils.countMatches(diffText, "diff-addedline"); + int deletedLines = StringUtils.countMatches(diffText, "diff-deletedline"); + int inlineChanges = StringUtils.countMatches(diffText, "diffchange-inline"); + + // Finally calculation of change Coefficient + double t = ((0.6 * deletedLines + 0.4 * addedLines) ) / previousLength; + changeCoefficient = 1 / exp(t); + + LOG.info("Change coefficient of revision {} (+{}-{}/{}): {}", revision.getLangRevId(), addedLines, + deletedLines, previousLength, changeCoefficient); + } - double t = ((0.6 * deletedLines + 0.4 * addedLines) ) / previousLength; - return 1 / (1 + exp(-t)); + return changeCoefficient; } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RevisionMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RevisionMediaWikiService.java index 5131df0..f0989d1 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RevisionMediaWikiService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RevisionMediaWikiService.java @@ -1,92 +1,89 @@ package org.wikitolearn.wikirating.service.mediawiki; import com.fasterxml.jackson.core.type.TypeReference; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.stereotype.Service; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikitolearn.wikirating.exception.GetDiffPreviousRevisionExeception; import org.wikitolearn.wikirating.model.Revision; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; /** * * @author aletundo, valsdav * */ @Service public class RevisionMediaWikiService extends MediaWikiService{ /** * Get the revision * @param apiUrl - * @param revId * @return the revision with the diff information * @throws GetDiffPreviousRevisionExeception */ - public String getDiffPreviousRevision(String apiUrl, int revId, int pageId) + public String getDiffPreviousRevision(String apiUrl, int oldRevId, int newRevId) throws GetDiffPreviousRevisionExeception { try { - Map parameters = mediaWikiApiUtils.getDiffRevisionParams(revId); + Map parameters = mediaWikiApiUtils.getDiffRevisionParams(oldRevId, newRevId); ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); InputStream response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); - String diffText = responseJson.getJSONObject("query").getJSONObject("pages") - .getJSONObject(Integer.toString(pageId)).getJSONArray("revisions").getJSONObject(0) - .getJSONObject("diff").getString("*"); + String diffText = responseJson.getJSONObject("compare").getString("*"); return diffText; } catch (JSONException e) { - LOG.error("An error occurred getting diff of previous revision of revision with id: {}. {}", revId, + LOG.error("An error occurred getting diff of previous revision of revision with id: {}. {}", newRevId, e.getMessage()); throw new GetDiffPreviousRevisionExeception(); } } /** * Get all the revisions for a specific page querying MediaWiki API * @param apiUrl the MediaWiki API url * @param pageId the id the page of which getting the revisions * @return revisions a list that contains all the fetched revisions */ public List getAllRevisionByPageId(String apiUrl, int pageId) { Map parameters = mediaWikiApiUtils.getRevisionParams(pageId); ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); InputStream response; boolean moreRevs = true; JSONArray revsJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List revs = new ArrayList<>(); try { while(moreRevs){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); toBeConcat.add(responseJson.getJSONObject("query").getJSONObject("pages"). getJSONObject(parameters.get("pageids")).getJSONArray("revisions")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("rvcontinue"); parameters.put("rvcontinue", continueFrom); }else{ moreRevs = false; revsJson = concatArrays(toBeConcat); } } revs = mapper.readValue(revsJson.toString(), new TypeReference>(){}); return revs; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return revs; } } diff --git a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java index de2a499..0f63d25 100644 --- a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java +++ b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java @@ -1,247 +1,246 @@ /** * */ package org.wikitolearn.wikirating.util; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; import java.util.Map; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikidata.wdtk.wikibaseapi.LoginFailedException; import org.wikitolearn.wikirating.exception.GenericException; /** * @author aletundo, valsdav * */ @Component public class MediaWikiApiUtils { private static final Logger LOG = LoggerFactory.getLogger(MediaWikiApiUtils.class); @Value("${mediawiki.api.user}") private String apiUser; @Value("${mediawiki.api.password}") private String apiPassword; /** * This method creates a MediaWiki Connection object * @param apiUrl the url of the API * @return ApiConnection the API connection object */ public ApiConnection getApiConnection(String apiUrl) { ApiConnection connection = new ApiConnection(apiUrl); if(connection.isLoggedIn()){ return connection; }else{ try { connection.login(apiUser, apiPassword); } catch (LoginFailedException e) { LOG.error("MediaWiki login failed. {}", e.getMessage()); // TODO change exception throw new GenericException(e.getMessage()); } return connection; } } /** * This method constructs the Map of parameters to attach with the MediaWiki Query to fetch all the pages * in the specified namespace * @param namespace The namespace whose pages are requested * @return Map having parameters */ public Map getListAllPagesParams(String namespace) { Map queryParameterMap = new HashMap<>(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allpages"); queryParameterMap.put("aplimit", "max"); queryParameterMap.put("apnamespace", namespace); queryParameterMap.put("apfilterredir", "nonredirects"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * Get a map * @param pid The PageID of the page for which revisions are requested * @return Map having parameters */ - public Map getDiffRevisionParams(int revId) { + public Map getDiffRevisionParams(int oldRevId, int newRevId) { Map queryParameterMap = new HashMap<>(); - queryParameterMap.put("action", "query"); - queryParameterMap.put("prop", "revisions"); - queryParameterMap.put("revids", Integer.toString(revId)); - queryParameterMap.put("rvdiffto", "prev"); + queryParameterMap.put("action", "compare"); + queryParameterMap.put("fromrev", Integer.toString(oldRevId)); + queryParameterMap.put("torev", Integer.toString(newRevId)); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to fetch all the revisions * of the given page * @param pid The PageID of the page for which revisions are requested * @return Map having parameters */ public Map getRevisionParams(int pid) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("prop", "revisions"); queryParameterMap.put("pageids", Integer.toString(pid)); queryParameterMap.put("rvprop", "userid|ids|timestamp|flags|size"); queryParameterMap.put("rvlimit", "max"); queryParameterMap.put("rvdir", "newer"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to get * all the users. * @return Map having parameters */ public Map getUserParams() { Map queryParameterMap = new HashMap<>(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allusers"); queryParameterMap.put("aulimit", "max"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach to the Mediawiki Query to get * all the recent changes in one namespace between two dates. * @param namespace namespace to user * @param begin start of the recentchanges * @param end end of the changes * @return the map with the parameters */ public Map getRecentChangesParams(String namespace, Date begin, Date end){ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); Map queryParameterMap = new HashMap<>(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "recentchanges"); queryParameterMap.put("rclimit", "max"); queryParameterMap.put("format", "json"); queryParameterMap.put("rcnamespace", namespace); queryParameterMap.put("rcshow", "!bot|!redirect"); queryParameterMap.put("rcprop", "title|userid|timestamp|ids|sizes|flags"); queryParameterMap.put("rctype", "new|edit"); queryParameterMap.put("rcstart", dateFormat.format(begin)); queryParameterMap.put("rcend", dateFormat.format(end)); queryParameterMap.put("rcdir", "newer"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach to the Mediawiki Query to get * all the log entries between two dates. We need the logs about moved pages, new users, deleted pages. * @param logtype Type of log to fetch: newusers|delete|move * @param begin start of the recentchanges * @param end end of the changes * @return the map with the parameters */ public Map getLogEventsParams(String logtype, Date begin, Date end){ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); Map queryParameterMap = new HashMap<>(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "logevents"); queryParameterMap.put("lelimit", "max"); queryParameterMap.put("lestart", dateFormat.format(begin)); queryParameterMap.put("leend", dateFormat.format(end)); queryParameterMap.put("ledir", "newer"); queryParameterMap.put("letype", logtype); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the Map of parameters to get the course tree structure * @param pageTitle the title of the page * @return the map with the parameters */ public Map getCourseTreeParams(String pageTitle){ Map courseTreeParameterMap = new HashMap<>(); courseTreeParameterMap.put("action", "coursetree"); courseTreeParameterMap.put("coursetitle", pageTitle); courseTreeParameterMap.put("format", "json"); return courseTreeParameterMap; } /** * This method converts an InputStream object to String * @param inputStream InputStream object to be converted * @return String result the converted stream into a string */ public JSONObject streamToJson(InputStream inputStream) { String result = ""; BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); } catch (UnsupportedEncodingException e1) { LOG.error("Unsupported enconding. {}", e1.getMessage()); } StringBuilder builder = new StringBuilder(); String line; try { if(reader!=null){ while ((line = reader.readLine()) != null) { builder.append(line); } } result = builder.toString(); inputStream.close(); } catch (IOException e) { LOG.error("An error occurs while reading the inputStream. {}", e.getMessage()); } JSONObject jsonResponse = null; try { jsonResponse = new JSONObject(result); } catch (JSONException e) { LOG.error("An error occurs while converting string to JSONObject. {}", e.getMessage()); } return jsonResponse; } /** * This method sends a request to MediaWiki API and then gets back an InputStream * @param connection ApiConnection The ApiConnection object * @param requestMethod RequestMethod The request method (ex: GET, POST, ...) * @param queryParametersMap Map The HashMap having all the query parameters * @return response InputStream The result data */ public InputStream sendRequest(ApiConnection connection, String requestMethod, Map queryParametersMap) { InputStream response = null; try { response = connection.sendRequest(requestMethod, queryParametersMap); } catch (IOException e) { LOG.error("Failed to send a request to MediaWiki API. {}", e.getMessage()); } return response; } }