diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index 982cf0e..4aab0c1 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,126 +1,128 @@ /** * */ package org.wikitolearn.wikirating.service; import java.util.Date; import java.util.List; import java.util.ListIterator; import java.util.Set; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.RevisionNotFoundException; import org.wikitolearn.wikirating.model.Page; import org.wikitolearn.wikirating.model.Revision; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; +import sun.rmi.runtime.Log; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private PageRepository pageRepository; /** * Initialize the revisions for the first time querying the MediaWiki API. * This method adds the revisions and sets the FIRST_REVISION, * LAST_REVISION and PREVIOUS_REVISION relationships. * @param lang the domain language * @param apiUrl the MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture initRevisions(String lang, String apiUrl) { + LOG.warn("LANG: {}", lang); Iterable pages = pageRepository.findAllByLang(lang); for(Page page : pages){ List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageid()); // Set the first and the last revisions for the current page page.setFistRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); ListIterator it = revisions.listIterator(); while(it.hasNext()){ Revision rev = it.next(); rev.setLangRevId(lang + "_" + rev.getRevid()); rev.setLang(lang); if (it.previousIndex() != 0){ rev.setPreviousRevision(revisions.get(it.previousIndex()-1)); } } // Saving all the revisions node and the page node revisionRepository.save(revisions); pageRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); } return CompletableFuture.completedFuture(true); } /** * Add a new Revision to the graph * @param revid * @param lang * @param userid * @param parentid * @param length * @param timestamp * @return */ public Revision addRevision(int revid, String lang, int userid, int parentid, int length, Date timestamp){ Revision rev = new Revision(revid, lang,userid, parentid, length, timestamp); revisionRepository.save(rev); return rev; } /** * Delete a revision given its langPageId * @param langPageId the langPageId of the revision */ public void deleteRevisionsOfPage(String langPageId) throws RevisionNotFoundException{ Set revisions = revisionRepository.findAllRevisionOfPage(langPageId); if (revisions.size() == 0){ LOG.error("Revisions of page {} not found", langPageId); throw new RevisionNotFoundException("Revisions of page "+langPageId+" not found"); } revisionRepository.delete(revisions); } /** * Get the requested revision * @param langRevId the langRevId of the revision * @return the revision * @throws RevisionNotFoundException */ public Revision getRevision(String langRevId) throws RevisionNotFoundException{ Revision revision = revisionRepository.findByLangRevId(langRevId); if(revision == null){ LOG.error("Revision {} not found", langRevId); throw new RevisionNotFoundException(); } return revision; } /** * Update the given revision * @param revision * @return the updated revision */ public Revision updateRevision(Revision revision){ revisionRepository.save(revision); return revision; } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/MediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/MediaWikiService.java index de4a869..017fa7f 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/MediaWikiService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/MediaWikiService.java @@ -1,52 +1,53 @@ /** * */ package org.wikitolearn.wikirating.service.mediawiki; import java.util.List; +import java.util.Map; import org.json.JSONArray; import org.json.JSONException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.wikitolearn.wikirating.util.MediaWikiApiUtils; import com.fasterxml.jackson.databind.ObjectMapper; /** * @author aletundo * @param * */ public abstract class MediaWikiService { protected final Logger LOG = LoggerFactory.getLogger(getClass()); @Autowired protected MediaWikiApiUtils mediaWikiApiUtils; @Autowired protected ObjectMapper mapper; @Value("${mediawiki.namespace}") protected String namespace; - public abstract List getAll(String apiUrl); + //public abstract List getAll(String apiUrl, Map parameters); /** * This method is an utility. It concatenates the given JSONArrays into one. * @param arrays List The arrays to be concatenated * @return result JSONArray The resulted JSONArray * @throws JSONException */ protected JSONArray concatArrays(List arrays) throws JSONException{ JSONArray result = new JSONArray(); for (JSONArray arr : arrays) { for (int i = 0; i < arr.length(); i++) { result.put(arr.get(i)); } } return result; } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/PageMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/PageMediaWikiService.java index 34b86ed..eb0d875 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/PageMediaWikiService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/PageMediaWikiService.java @@ -1,107 +1,107 @@ /** * */ package org.wikitolearn.wikirating.service.mediawiki; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.stereotype.Service; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikitolearn.wikirating.exception.GenericException; import org.wikitolearn.wikirating.model.CourseTree; import org.wikitolearn.wikirating.model.Page; import com.fasterxml.jackson.core.type.TypeReference; /** * * @author aletundo, valsdav * */ @Service public class PageMediaWikiService extends MediaWikiService{ /** * Get all the pages from a specified namespace of MediaWiki instance through its API. * @param apiUrl String The MediaWiki API url * @return pages List A list that contains all the fetched pages */ - @Override + //@Override public List getAll(String apiUrl){ ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); Map parameters = mediaWikiApiUtils.getListAllPagesParams(namespace); InputStream response; boolean morePages = true; JSONArray pagesJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List pages = new ArrayList<>(); try { while(morePages){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("allpages")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("apcontinue"); parameters.put("apfrom", continueFrom); }else{ morePages = false; pagesJson = concatArrays(toBeConcat); } } pages = mapper.readValue(pagesJson.toString(), new TypeReference>(){}); return pages; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return pages; } /** * Get the course tree structure through the MediaWiki API * @param apiUrl the MediaWiki API url * @param pageTitle the title of the root course page * @return the course tree */ public CourseTree getCourseTree(String apiUrl, String pageTitle) { ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); Map parameters = mediaWikiApiUtils.getCourseTreeParams(pageTitle); InputStream response; response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); try { JSONObject jsonTree = responseJson.getJSONObject("coursetree").getJSONObject("response"); List> levelsThree = new ArrayList<>(); JSONArray levelsThreeJson = jsonTree.getJSONArray("levelsThree"); levelsThree = mapper.readValue(levelsThreeJson.toString(), new TypeReference>>(){}); // Build course tree manually cause difficulties with serialization of nested JSON arrays CourseTree courseTree = new CourseTree(); courseTree.setRoot(jsonTree.getString("root")); courseTree.setLevelsTwo( mapper.readValue( jsonTree.getJSONArray("levelsTwo").toString(), new TypeReference>(){} ) ); courseTree.setLevelsTree(levelsThree); LOG.info("Got course tree for page {}: {}", pageTitle, courseTree.toString()); return courseTree; } catch (JSONException | IOException e) { LOG.error("An error occurred: {}", e.getMessage()); throw new GenericException(e.getMessage()); } } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RevisionMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RevisionMediaWikiService.java index b4490e5..ed836cb 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RevisionMediaWikiService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/RevisionMediaWikiService.java @@ -1,78 +1,79 @@ package org.wikitolearn.wikirating.service.mediawiki; import com.fasterxml.jackson.core.type.TypeReference; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.stereotype.Service; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikitolearn.wikirating.model.Revision; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; /** * * @author aletundo, valsdav * */ @Service public class RevisionMediaWikiService extends MediaWikiService{ - - private Map parameters; /** * Get all the revisions of a page. * @param apiUrl String The MediaWiki API url * @return revisions List A list that contains all the fetched revisions */ - @Override - public List getAll(String apiUrl){ + //@Override + public List getAll(String apiUrl,Map parameters){ ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); InputStream response; boolean moreRevs = true; JSONArray revsJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List revs = new ArrayList<>(); try { while(moreRevs){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); + LOG.info(responseJson.toString()); toBeConcat.add(responseJson.getJSONObject("query").getJSONObject("pages"). getJSONObject(parameters.get("pageids")).getJSONArray("revisions")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("rvcontinue"); parameters.put("rvcontinue", continueFrom); }else{ moreRevs = false; revsJson = concatArrays(toBeConcat); } } revs = mapper.readValue(revsJson.toString(), new TypeReference>(){}); return revs; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return revs; } /** * Get all the revisions for a specific page querying MediaWiki API * @param apiUrl String The MediaWiki API url - * @param pageid int The id the page of which getting the revisions + * @param pageId int The id the page of which getting the revisions * @return revisions List A list that contains all the fetched revisions */ public List getAllRevisionByPageId(String apiUrl, int pageId) { - parameters = mediaWikiApiUtils.getRevisionParams(pageId); - return getAll(apiUrl); + LOG.info("ApiUrl: {}",apiUrl); + LOG.info("PageId {}", pageId); + Map parameters = mediaWikiApiUtils.getRevisionParams(pageId); + return getAll(apiUrl, parameters); } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java index 32c5142..a134f27 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UpdateMediaWikiService.java @@ -1,152 +1,152 @@ package org.wikitolearn.wikirating.service.mediawiki; import com.fasterxml.jackson.core.type.TypeReference; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.stereotype.Service; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikitolearn.wikirating.model.UpdateInfo; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; /** * Created by valsdav on 29/03/17. */ @Service public class UpdateMediaWikiService extends MediaWikiService{ /** * * @param apiUrl * @param namespace * @param start * @param end * @return */ public List getPagesUpdateInfo(String apiUrl, String namespace, Date start, Date end){ List editsAndNewPages = getRecentChangesBetweenDates(apiUrl, namespace, start, end); List deletedPages = getLogEventsBetweenDates(apiUrl, "delete", start, end); List movedPages = getLogEventsBetweenDates(apiUrl, "move", start, end); List allUpdates = Stream.of(editsAndNewPages,deletedPages,movedPages) .flatMap(List::stream).collect(Collectors.toList()); return allUpdates; } /** * * @param apiUrl * @param start * @param end * @return */ public List getNewUsers(String apiUrl, Date start, Date end){ return getLogEventsBetweenDates(apiUrl, "newusers", start, end); } /** * * @param apiUrl * @param namespace * @param start * @param end * @return */ public List getRecentChangesBetweenDates( String apiUrl, String namespace, Date start, Date end){ ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); InputStream response; boolean moreRecentChanges = true; JSONArray recentChangesJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List recentChanges = new ArrayList<>(); Map parameters = mediaWikiApiUtils.getRecentChangesParams(namespace, start,end); try { while(moreRecentChanges){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("recentchanges")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("rccontinue"); parameters.put("rccontinue", continueFrom); }else{ moreRecentChanges = false; recentChangesJson = concatArrays(toBeConcat); } } recentChanges = mapper.readValue(recentChangesJson.toString(), new TypeReference>(){}); return recentChanges; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return recentChanges; } /** * * @param apiUrl * @param logtype * @param start * @param end * @return */ public List getLogEventsBetweenDates( String apiUrl, String logtype, Date start, Date end){ ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); InputStream response; JSONArray logEventsJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List logEvents = new ArrayList<>(); try { Map parameters = mediaWikiApiUtils.getLogEventsParams(logtype, start,end); boolean moreLogEvents = true; while(moreLogEvents){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("recentchanges")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("lecontinue"); parameters.put("lecontinue", continueFrom); }else{ moreLogEvents = false; logEventsJson = concatArrays(toBeConcat); } } if(logtype.equals("move")){ for (int i = 0; i < logEventsJson.length(); i++){ JSONObject element = logEventsJson.getJSONObject(i); String newTitle = (String) element.getJSONObject("params").get("target_title"); element.put("newTitle", newTitle); } } logEvents = mapper.readValue(logEventsJson.toString(), new TypeReference>(){}); return logEvents; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return logEvents; } /** * */ - @Override + //@Override public List getAll(String apiUrl) { return null; } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UserMediaWikiService.java b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UserMediaWikiService.java index 437601d..d268e05 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UserMediaWikiService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/mediawiki/UserMediaWikiService.java @@ -1,64 +1,64 @@ package org.wikitolearn.wikirating.service.mediawiki; import com.fasterxml.jackson.core.type.TypeReference; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.springframework.stereotype.Service; import org.wikidata.wdtk.wikibaseapi.ApiConnection; import org.wikitolearn.wikirating.model.User; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.List; import java.util.Map; /** * * @author aletundo, valsdav * */ @Service public class UserMediaWikiService extends MediaWikiService{ /** * Get all the users from MediaWiki instance through its API. * @param apiUrl String The MediaWiki API url * @return users List A list that contains all the fetched users */ - @Override + //@Override public List getAll(String apiUrl){ ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); Map parameters = mediaWikiApiUtils.getUserParams(); InputStream response; boolean moreUsers = true; JSONArray usersJson = new JSONArray(); List toBeConcat = new ArrayList<>(); List users = new ArrayList<>(); try { while(moreUsers){ response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("allusers")); if(responseJson.has("continue")){ String continueFrom = responseJson.getJSONObject("continue").getString("aufrom"); parameters.put("aufrom", continueFrom); }else{ moreUsers = false; usersJson = concatArrays(toBeConcat); } } users = mapper.readValue(usersJson.toString(), new TypeReference>(){}); return users; } catch (JSONException e){ LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); } catch(IOException e){ LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); } return users; } }