diff --git a/src/main/java/org/wikitolearn/wikirating/model/UpdateEvent.java b/src/main/java/org/wikitolearn/wikirating/model/UpdateEvent.java new file mode 100644 index 0000000..611638e --- /dev/null +++ b/src/main/java/org/wikitolearn/wikirating/model/UpdateEvent.java @@ -0,0 +1,112 @@ +package org.wikitolearn.wikirating.model; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.wikitolearn.wikirating.util.enums.UpdateType; + +import java.util.Date; +import java.util.Map; + +/** + * This class represents a RecentChange entry during the fetch Process. + * It stores the information fetched from the API before the update of the DB. + * Created by valsdav on 30/03/17. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class UpdateEvent { + + private UpdateType type; + private int pageid; + private int revid; + private int old_revid; + private int userid; + private int oldlen; + private int newlen; + private UpdateType logtype; + private Map logparams; + private Date timestamp; + + public UpdateEvent(){} + + public UpdateType getType() { + if (this.type == UpdateType.LOG){ + return this.logtype; + } + return type; + } + + public void setType(UpdateType type) { + this.type = type; + } + + public int getPageid() { + return pageid; + } + + public void setPageid(int pageid) { + this.pageid = pageid; + } + + public int getRevid() { + return revid; + } + + public void setRevid(int revid) { + this.revid = revid; + } + + public int getOld_revid() { + return old_revid; + } + + public void setOld_revid(int old_revid) { + this.old_revid = old_revid; + } + + public int getUserid() { + return userid; + } + + public void setUserid(int userid) { + this.userid = userid; + } + + public int getOldlen() { + return oldlen; + } + + public void setOldlen(int oldlen) { + this.oldlen = oldlen; + } + + public int getNewlen() { + return newlen; + } + + public void setNewlen(int newlen) { + this.newlen = newlen; + } + + public Date getTimestamp() { + return timestamp; + } + + public void setTimestamp(Date timestamp) { + this.timestamp = timestamp; + } + + public UpdateType getLogtype() { + return logtype; + } + + public void setLogtype(UpdateType logtype) { + this.logtype = logtype; + } + + public Map getLogparams() { + return logparams; + } + + public void setLogparams(Map logparams) { + this.logparams = logparams; + } +} diff --git a/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java new file mode 100644 index 0000000..4f7332f --- /dev/null +++ b/src/main/java/org/wikitolearn/wikirating/service/UpdateService.java @@ -0,0 +1,52 @@ +package org.wikitolearn.wikirating.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; +import org.wikitolearn.wikirating.model.Process; +import org.wikitolearn.wikirating.model.UpdateEvent; +import org.wikitolearn.wikirating.service.mediawiki.RecentChangesMediaWikiService; +import org.wikitolearn.wikirating.util.enums.ProcessType; + +import java.util.Date; +import java.util.List; + +/** + * Created by valsdav on 29/03/17. + */ +@Service +public class UpdateService { + private static final Logger LOG = LoggerFactory.getLogger(UpdateService.class); + + @Autowired private PageService pageService; + @Autowired private UserService userService; + @Autowired private RevisionService revisionService; + @Autowired private ProcessService processService; + @Autowired private MetadataService metadataService; + @Autowired private RecentChangesMediaWikiService recentChangesMediaWikiService; + @Value("#{'${mediawiki.langs}'.split(',')}") + private List langs; + @Value("${mediawiki.protocol}") + private String protocol; + @Value("${mediawiki.api.url}") + private String apiUrl; + + private boolean updateData(){ + //getting the begin timestamp of the latest FETCH Process before opening a new process + Date beginOfLastFetch = processService.getLastProcessStartDateByType(ProcessType.FETCH); + //opening a new FETCH process + Process currentFetchProcess = processService.createNewProcess(ProcessType.FETCH); + //First of all we get the RecentChangeEvents from Mediawiki API. + String url = protocol + langs.get(0) + "." + apiUrl; + List updates = recentChangesMediaWikiService.getRecentChangesBetweenDates(apiUrl,"2800", + beginOfLastFetch, currentFetchProcess.getStartOfProcess()); + + + return true; + } + + + +} diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RecentChangesMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RecentChangesMediaWikiService.java new file mode 100644 index 0000000..a34260d --- /dev/null +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RecentChangesMediaWikiService.java @@ -0,0 +1,62 @@ +package org.wikitolearn.wikirating.service.mediawiki; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.stereotype.Service; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikitolearn.wikirating.model.UpdateEvent; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; + +/** + * Created by valsdav on 29/03/17. + */ +@Service +public class RecentChangesMediaWikiService extends MediaWikiService{ + + public List getRecentChangesBetweenDates(String apiUrl, String namespace, Date begin, Date end){ + ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); + InputStream response; + boolean moreRc = true; + JSONArray rcJson = new JSONArray(); + List toBeConcat = new ArrayList<>(); + List rcs = new ArrayList<>(); + Map parameters = mediaWikiApiUtils.getRecentChangesParam(namespace, begin,end); + try { + while(moreRc){ + response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); + JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); + + toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("recentchanges")); + + if(responseJson.has("continue")){ + String continueFrom = responseJson.getJSONObject("continue").getString("rccontinue"); + parameters.put("rccontinue", continueFrom); + }else{ + moreRc = false; + rcJson = concatArrays(toBeConcat); + } + } + rcs = mapper.readValue(rcJson.toString(), new TypeReference>(){}); + return rcs; + } catch (JSONException e){ + LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); + } catch(IOException e){ + LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); + } + return rcs; + + } + + @Override + public List getAll(String apiUrl) { + return null; + } +} diff --git a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java index 1e897d3..9ad7675 100644 --- a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java +++ b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java @@ -1,170 +1,198 @@ /** * */ package org.wikitolearn.wikirating.util; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; +import java.text.SimpleDateFormat; +import java.time.format.DateTimeFormatter; +import java.util.Date; import java.util.HashMap; import java.util.Map; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikidata.wdtk.wikibaseapi.LoginFailedException; import org.wikitolearn.wikirating.exception.GenericException; /** * @author aletundo, valsdav * */ @Component public class MediaWikiApiUtils { private static final Logger LOG = LoggerFactory.getLogger(MediaWikiApiUtils.class); @Value("${mediawiki.api.user}") private String apiUser; @Value("${mediawiki.api.password}") private String apiPassword; /** * This method creates a MediaWiki Connection object * @param apiUrl the url of the API * @return ApiConnection the API connection object */ public ApiConnection getApiConnection(String apiUrl) { ApiConnection connection = new ApiConnection(apiUrl); if(connection.isLoggedIn()){ return connection; }else{ try { connection.login(apiUser, apiPassword); } catch (LoginFailedException e) { LOG.error("MediaWiki login failed. {}", e.getMessage()); // TODO change exception throw new GenericException(e.getMessage()); } return connection; } } /** * This method constructs the Map of parameters to attach with the MediaWiki Query to fetch all the pages * in the specified namespace * @param namespace The namespace whose pages are requested * @return Map having parameters */ public Map getListAllPagesParamsMap(String namespace) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allpages"); queryParameterMap.put("aplimit", "max"); queryParameterMap.put("apnamespace", namespace); queryParameterMap.put("apfilterredir", "nonredirects"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to fetch all the revisions * of the given page * @param pid The PageID of the page for which revisions are requested * @return Map having parameters */ public Map getRevisionParam(int pid) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("prop", "revisions"); queryParameterMap.put("pageids", Integer.toString(pid)); queryParameterMap.put("rvprop", "userid|ids|timestamp|flags|size"); queryParameterMap.put("rvlimit", "max"); queryParameterMap.put("rvdir", "newer"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to get * all the users. * @return Map having parameters */ public Map getUserParam() { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allusers"); queryParameterMap.put("aulimit", "max"); queryParameterMap.put("format", "json"); return queryParameterMap; } + + /** + * This method constructs the MAP of parameters to attach iwth the Mediawiki Query to get + * all the recent changes in one namespace between two dates. + * @param namespace namespace to user + * @param begin start of the recentchanges + * @param end end of the changes + * @return the map with the parameters + */ + public Map getRecentChangesParam(String namespace, Date begin, Date end){ + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + Map queryParameterMap = new HashMap<>(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("list", "recentchanges"); + queryParameterMap.put("rclimit", "max"); + queryParameterMap.put("format", "json"); + queryParameterMap.put("rcnamespace", namespace); + queryParameterMap.put("rcshow", "!bot|!redirect"); + queryParameterMap.put("rcprop", "userid|timestamp|ids|sizes|flags|loginfo"); + queryParameterMap.put("rctype", "new|edit|log"); + queryParameterMap.put("rcstart", dateFormat.format(begin)); + queryParameterMap.put("rcend", dateFormat.format(end)); + queryParameterMap.put("rcdir", "newer"); + return queryParameterMap; + } /** * This method converts an InputStream object to String * @param inputStream InputStream object to be converted * @return String result the converted stream into a string */ public JSONObject streamToJson(InputStream inputStream) { String result = ""; BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); } catch (UnsupportedEncodingException e1) { LOG.error("Unsupported enconding. {}", e1.getMessage()); } StringBuilder builder = new StringBuilder(); String line; try { if(reader!=null){ while ((line = reader.readLine()) != null) { builder.append(line); } } result = builder.toString(); inputStream.close(); } catch (IOException e) { LOG.error("An error occurs while reading the inputStream. {}", e.getMessage()); } JSONObject jsonResponse = null; try { jsonResponse = new JSONObject(result); } catch (JSONException e) { LOG.error("An error occurs while converting string to JSONObject. {}", e.getMessage()); } return jsonResponse; } /** * This method sends a request to MediaWiki API and then gets back an InputStream * @param connection ApiConnection The ApiConnection object * @param requestMethod RequestMethod The request method (ex: GET, POST, ...) * @param queryParametersMap Map The HashMap having all the query parameters * @return response InputStream The result data */ public InputStream sendRequest(ApiConnection connection, String requestMethod, Map queryParametersMap) { InputStream response = null; try { response = connection.sendRequest(requestMethod, queryParametersMap); } catch (IOException e) { LOG.error("Failed to send a request to MediaWiki API. {}", e.getMessage()); } return response; } } diff --git a/src/main/java/org/wikitolearn/wikirating/util/enums/UpdateType.java b/src/main/java/org/wikitolearn/wikirating/util/enums/UpdateType.java new file mode 100644 index 0000000..c84b725 --- /dev/null +++ b/src/main/java/org/wikitolearn/wikirating/util/enums/UpdateType.java @@ -0,0 +1,8 @@ +package org.wikitolearn.wikirating.util.enums; + +/** + * Created by valsdav on 30/03/17. + */ +public enum UpdateType { + NEW, EDIT, MOVE, DELETE, LOG; +}