diff --git a/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java b/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java index 3d8f16c..ff314a3 100644 --- a/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java +++ b/src/main/java/org/wikitolearn/wikirating/model/UpdateInfo.java @@ -1,108 +1,117 @@ package org.wikitolearn.wikirating.model; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.wikitolearn.wikirating.util.enums.UpdateType; import java.util.Date; /** * This class represents a RecentChange entry during the fetch Process. * It stores the information fetched from the API before the update of the DB. * Created by valsdav on 30/03/17. */ @JsonIgnoreProperties(ignoreUnknown = true) public class UpdateInfo { private UpdateType type; private String title; + private String newTitle; private int pageid; private int revid; private int old_revid; private String user; private int userid; private int oldlen; private int newlen; private Date timestamp; public UpdateInfo(){} public UpdateType getType() { return type; } public void setType(UpdateType type) { this.type = type; } public int getPageid() { return pageid; } public void setPageid(int pageid) { this.pageid = pageid; } public int getRevid() { return revid; } public void setRevid(int revid) { this.revid = revid; } public int getOld_revid() { return old_revid; } public void setOld_revid(int old_revid) { this.old_revid = old_revid; } public int getUserid() { return userid; } public void setUserid(int userid) { this.userid = userid; } public int getOldlen() { return oldlen; } public void setOldlen(int oldlen) { this.oldlen = oldlen; } public int getNewlen() { return newlen; } public void setNewlen(int newlen) { this.newlen = newlen; } public Date getTimestamp() { return timestamp; } public void setTimestamp(Date timestamp) { this.timestamp = timestamp; } public String getTitle() { return title; } public void setTitle(String title) { this.title = title; } public String getUser() { return user; } public void setUser(String user) { this.user = user; } + + public String getNewTitle() { + return newTitle; + } + + public void setNewTitle(String newTitle) { + this.newTitle = newTitle; + } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java index 3019835..b516c74 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java @@ -1,122 +1,128 @@ package org.wikitolearn.wikirating.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.model.Process; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.model.UpdateInfo; import org.wikitolearn.wikirating.model.User; import org.wikitolearn.wikirating.service.mediawiki.UpdateMediaWikiService; +import org.wikitolearn.wikirating.util.enums.ProcessStatus; import org.wikitolearn.wikirating.util.enums.ProcessType; import java.util.ArrayList; import java.util.Date; import java.util.List; /** * Created by valsdav on 29/03/17. */ @Service public class UpdateService { private static final Logger LOG = LoggerFactory.getLogger(UpdateService.class); @Autowired private UserService userService; @Autowired private PageService pageService; @Autowired private RevisionService revisionService; @Autowired private ProcessService processService; @Autowired private UpdateMediaWikiService updateMediaWikiService; @Value("#{'${mediawiki.langs}'.split(',')}") private List langs; @Value("${mediawiki.protocol}") private String protocol; @Value("${mediawiki.api.url}") private String apiUrl; @Value("${mediawiki.namespace}") private String namespace; /** * * @return */ public boolean updateGraph() { // Get start timestamp of the latest FETCH Process before opening a new process Date startTimestampLatestFetch = processService.getLastProcessStartDateByType(ProcessType.FETCH); // Create a new FETCH process Process currentFetchProcess = processService.createNewProcess(ProcessType.FETCH); LOG.info("Created new fetch process {}", currentFetchProcess.toString()); Date startTimestampCurrentFetch = currentFetchProcess.getStartOfProcess(); updatePagesAndRevisions(startTimestampLatestFetch, startTimestampCurrentFetch); updateUsers(startTimestampLatestFetch, startTimestampCurrentFetch); + + // Save the result of the process, closing the current one + processService.closeCurrentProcess(ProcessStatus.DONE); return true; } /** * @param start * @param end */ private void updateUsers(Date start, Date end) { String url = protocol + langs.get(0) + "." + apiUrl; List usersUpdateInfo = updateMediaWikiService.getNewUsers(url, start, end); List newUsers = new ArrayList<>(); // Build a list with new users to be added to the graph for(UpdateInfo updateInfo : usersUpdateInfo){ User user = new User(); user.setUserId(updateInfo.getUserid()); user.setUsername(updateInfo.getUser()); newUsers.add(user); } userService.addUsers(newUsers); } /** * * @param start * @param end */ private void updatePagesAndRevisions(Date start, Date end) { // First of all, get the RecentChangeEvents from MediaWiki API for (String lang : langs) { String url = protocol + lang + "." + apiUrl; // Fetching pages updates List updates = updateMediaWikiService.getPagesUpdateInfo(url, namespace, start, end); updates.forEach(update -> { switch (update.getType()) { case NEW: // Create the new revision Revision newRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Then create a new Page and link it with the revision pageService.addPage(update.getPageid(), update.getTitle(), lang, newRev); break; case EDIT: // Create a new revision Revision updateRev = revisionService.addRevision(update.getRevid(), lang, update.getUserid(), update.getOld_revid(), update.getNewlen(), update.getTimestamp()); // Then add it to the page pageService.addRevisionToPage(lang + "_" + update.getPageid(), updateRev); break; case MOVE: + // Move the page to the new title + pageService.movePage(update.getTitle(), update.getNewTitle(), lang); break; case DELETE: break; default: break; } }); } } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java index e994e24..126c422 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java @@ -1,145 +1,152 @@ package org.wikitolearn.wikirating.service.mediawiki; import com.fasterxml.jackson.core.type.TypeReference; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.stereotype.Service; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikitolearn.wikirating.model.UpdateInfo; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; /** * Created by valsdav on 29/03/17. */ @Service public class UpdateMediaWikiService extends MediaWikiService{ - + /** * * @param apiUrl * @param namespace * @param start * @param end * @return */ public List getPagesUpdateInfo(String apiUrl, String namespace, Date start, Date end){ List editsAndNewPages = getRecentChangesBetweenDates(apiUrl, namespace, start, end); List deletedPages = getLogEventsBetweenDates(apiUrl, "delete", start, end); List movedPages = getLogEventsBetweenDates(apiUrl, "move", start, end); List allUpdates = Stream.of(editsAndNewPages,deletedPages,movedPages) .flatMap(List::stream).collect(Collectors.toList()); return allUpdates; } /** * * @param apiUrl * @param start * @param end * @return */ public List getNewUsers(String apiUrl, Date start, Date end){ return getLogEventsBetweenDates(apiUrl, "newusers", start, end); } /** * * @param apiUrl * @param namespace * @param start * @param end * @return */ public List getRecentChangesBetweenDates( String apiUrl, String namespace, Date start, Date end){ ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); InputStream response; boolean moreRecentChanges = true; JSONArray recentChangesJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List recentChanges = new ArrayList<>(); Map parameters = mediaWikiApiUtils.getRecentChangesParam(namespace, start,end); try { while(moreRecentChanges){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("recentchanges")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("rccontinue"); parameters.put("rccontinue", continueFrom); }else{ moreRecentChanges = false; recentChangesJson = concatArrays(toBeConcat); } } recentChanges = mapper.readValue(recentChangesJson.toString(), new TypeReference>(){}); return recentChanges; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return recentChanges; } /** * * @param apiUrl * @param logtype * @param start * @param end * @return */ public List getLogEventsBetweenDates( String apiUrl, String logtype, Date start, Date end){ ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); InputStream response; JSONArray logEventsJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List logEvents = new ArrayList<>(); try { Map parameters = mediaWikiApiUtils.getLogEventsParam(logtype, start,end); boolean moreLogEvents = true; while(moreLogEvents){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("recentchanges")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("lecontinue"); parameters.put("lecontinue", continueFrom); }else{ moreLogEvents = false; logEventsJson = concatArrays(toBeConcat); } } + if(logtype.equals("move")){ + for (int i = 0; i < logEventsJson.length(); i++){ + JSONObject element = logEventsJson.getJSONObject(i); + String newTitle = (String) element.getJSONObject("params").get("target_title"); + element.put("newTitle", newTitle); + } + } logEvents = mapper.readValue(logEventsJson.toString(), new TypeReference>(){}); return logEvents; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return logEvents; } /** - * + * */ @Override public List getAll(String apiUrl) { return null; } } diff --git a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java index 1623eb2..58e9a0b 100644 --- a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java +++ b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java @@ -1,220 +1,219 @@ /** * */ package org.wikitolearn.wikirating.util; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.text.SimpleDateFormat; import java.time.format.DateTimeFormatter; import java.util.Date; import java.util.HashMap; import java.util.Map; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikidata.wdtk.wikibaseapi.LoginFailedException; import org.wikitolearn.wikirating.exception.GenericException; /** * @author aletundo, valsdav * */ @Component public class MediaWikiApiUtils { private static final Logger LOG = LoggerFactory.getLogger(MediaWikiApiUtils.class); @Value("${mediawiki.api.user}") private String apiUser; @Value("${mediawiki.api.password}") private String apiPassword; /** * This method creates a MediaWiki Connection object * @param apiUrl the url of the API * @return ApiConnection the API connection object */ public ApiConnection getApiConnection(String apiUrl) { ApiConnection connection = new ApiConnection(apiUrl); if(connection.isLoggedIn()){ return connection; }else{ try { connection.login(apiUser, apiPassword); } catch (LoginFailedException e) { LOG.error("MediaWiki login failed. {}", e.getMessage()); // TODO change exception throw new GenericException(e.getMessage()); } return connection; } } /** * This method constructs the Map of parameters to attach with the MediaWiki Query to fetch all the pages * in the specified namespace * @param namespace The namespace whose pages are requested * @return Map having parameters */ public Map getListAllPagesParamsMap(String namespace) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allpages"); queryParameterMap.put("aplimit", "max"); queryParameterMap.put("apnamespace", namespace); queryParameterMap.put("apfilterredir", "nonredirects"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to fetch all the revisions * of the given page * @param pid The PageID of the page for which revisions are requested * @return Map having parameters */ public Map getRevisionParam(int pid) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("prop", "revisions"); queryParameterMap.put("pageids", Integer.toString(pid)); queryParameterMap.put("rvprop", "userid|ids|timestamp|flags|size"); queryParameterMap.put("rvlimit", "max"); queryParameterMap.put("rvdir", "newer"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to get * all the users. * @return Map having parameters */ public Map getUserParam() { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allusers"); queryParameterMap.put("aulimit", "max"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach to the Mediawiki Query to get * all the recent changes in one namespace between two dates. * @param namespace namespace to user * @param begin start of the recentchanges * @param end end of the changes * @return the map with the parameters */ public Map getRecentChangesParam(String namespace, Date begin, Date end){ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); Map queryParameterMap = new HashMap<>(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "recentchanges"); queryParameterMap.put("rclimit", "max"); queryParameterMap.put("format", "json"); queryParameterMap.put("rcnamespace", namespace); queryParameterMap.put("rcshow", "!bot|!redirect"); queryParameterMap.put("rcprop", "title|userid|timestamp|ids|sizes|flags"); queryParameterMap.put("rctype", "new|edit"); queryParameterMap.put("rcstart", dateFormat.format(begin)); queryParameterMap.put("rcend", dateFormat.format(end)); queryParameterMap.put("rcdir", "newer"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach to the Mediawiki Query to get * all the log entries between two dates. We need the logs about moved pages, new users, deleted pages. * @param logtype Type of log to fetch: newusers|delete|move * @param begin start of the recentchanges * @param end end of the changes * @return the map with the parameters */ public Map getLogEventsParam(String logtype, Date begin, Date end){ SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); Map queryParameterMap = new HashMap<>(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "logevents"); queryParameterMap.put("lelimit", "max"); queryParameterMap.put("format", "json"); - queryParameterMap.put("leprop", "ids|title|user|userid|timestamp"); queryParameterMap.put("lestart", dateFormat.format(begin)); queryParameterMap.put("leend", dateFormat.format(end)); queryParameterMap.put("ledir", "newer"); return queryParameterMap; } /** * This method converts an InputStream object to String * @param inputStream InputStream object to be converted * @return String result the converted stream into a string */ public JSONObject streamToJson(InputStream inputStream) { String result = ""; BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); } catch (UnsupportedEncodingException e1) { LOG.error("Unsupported enconding. {}", e1.getMessage()); } StringBuilder builder = new StringBuilder(); String line; try { if(reader!=null){ while ((line = reader.readLine()) != null) { builder.append(line); } } result = builder.toString(); inputStream.close(); } catch (IOException e) { LOG.error("An error occurs while reading the inputStream. {}", e.getMessage()); } JSONObject jsonResponse = null; try { jsonResponse = new JSONObject(result); } catch (JSONException e) { LOG.error("An error occurs while converting string to JSONObject. {}", e.getMessage()); } return jsonResponse; } /** * This method sends a request to MediaWiki API and then gets back an InputStream * @param connection ApiConnection The ApiConnection object * @param requestMethod RequestMethod The request method (ex: GET, POST, ...) * @param queryParametersMap Map The HashMap having all the query parameters * @return response InputStream The result data */ public InputStream sendRequest(ApiConnection connection, String requestMethod, Map queryParametersMap) { InputStream response = null; try { response = connection.sendRequest(requestMethod, queryParametersMap); } catch (IOException e) { LOG.error("Failed to send a request to MediaWiki API. {}", e.getMessage()); } return response; } }