diff --git a/src/main/java/org/wikitolearn/wikirating/WikiRatingApplication.java b/src/main/java/org/wikitolearn/wikirating/WikiRatingApplication.java index 89b27e0..deab5ba 100644 --- a/src/main/java/org/wikitolearn/wikirating/WikiRatingApplication.java +++ b/src/main/java/org/wikitolearn/wikirating/WikiRatingApplication.java @@ -1,39 +1,39 @@ package org.wikitolearn.wikirating; import java.util.concurrent.Executor; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.domain.EntityScan; import org.springframework.data.neo4j.repository.config.EnableNeo4jRepositories; import org.springframework.scheduling.annotation.AsyncConfigurerSupport; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; import org.springframework.transaction.annotation.EnableTransactionManagement; /** * * @author aletundo * */ @SpringBootApplication @EnableAsync @EntityScan("org.wikitolearn.wikirating.model") @EnableNeo4jRepositories(basePackages = "org.wikitolearn.wikirating.repository") @EnableTransactionManagement public class WikiRatingApplication extends AsyncConfigurerSupport{ public static void main(String[] args) { SpringApplication.run(WikiRatingApplication.class, args); } @Override public Executor getAsyncExecutor() { ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); - executor.setCorePoolSize(5); - executor.setMaxPoolSize(10); + executor.setCorePoolSize(10); + executor.setMaxPoolSize(20); executor.setThreadNamePrefix("ServicesThread-"); executor.initialize(); return executor; } } diff --git a/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java b/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java index 862103e..209e8e7 100644 --- a/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java +++ b/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java @@ -1,138 +1,197 @@ /** * */ package org.wikitolearn.wikirating.controller; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.util.DefaultPropertiesPersister; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; -import org.wikitolearn.wikirating.model.Process; +//import org.wikitolearn.wikirating.model.Process; import org.wikitolearn.wikirating.service.PageService; import org.wikitolearn.wikirating.service.RevisionService; import org.wikitolearn.wikirating.service.UserService; -import org.wikitolearn.wikirating.util.enums.ProcessResult; -import org.wikitolearn.wikirating.util.enums.ProcessType; +//import org.wikitolearn.wikirating.util.MediaWikiApiUtils; +//import org.wikitolearn.wikirating.util.enums.ProcessResult; +//import org.wikitolearn.wikirating.util.enums.ProcessType; /** * * @author aletundo * */ @RestController public class MaintenanceController { private static final Logger LOG = LoggerFactory.getLogger(MaintenanceController.class); - + @Autowired private PageService pageService; @Autowired private UserService userService; @Autowired - private RevisionService revisionService; - + private RevisionService revisionService; + @Value("#{'${mediawiki.langs}'.split(',')}") + private List langs; + @Value("${mediawiki.protocol}") + private String protocol; + @Value("${mediawiki.api.url}") + private String apiUrl; + /** - * Secured endpoint to enable or disable read only mode. When read only mode is enabled only GET requests are - * allowed. To change this behavior @see org.wikitolearn.wikirating.wikirating.filter.MaintenanceFilter. - * @param active String requested parameter that toggle the re. Binary values are accepted. + * Secured endpoint to enable or disable read only mode. When read only mode + * is enabled only GET requests are allowed. To change this behavior @see + * org.wikitolearn.wikirating.wikirating.filter.MaintenanceFilter. + * + * @param active + * String requested parameter that toggle the re. Binary values + * are accepted. * @return a JSON object with the response */ @RequestMapping(value = "${maintenance.readonlymode.uri}", method = RequestMethod.POST, produces = "application/json") public String toggleReadOnlyMode(@RequestParam(value = "active") String active) { JSONObject response = new JSONObject(); int mode = Integer.parseInt(active); try { if (mode == 0) { response.put("status", "success"); response.put("message", "Application is live now."); // Delete maintenance lock file if it exists File f = new File("maintenance.lock"); if (f.exists()) { f.delete(); LOG.debug("Deleted maintenance lock file."); } LOG.info("Application is live now."); } else if (mode == 1) { try { response.put("status", "success"); response.put("message", "Application is in maintenance mode now."); - // Create maintenance lock file with a maintenance.active property setted to true + // Create maintenance lock file with a maintenance.active + // property setted to true Properties props = new Properties(); props.setProperty("maintenance.active", "true"); File lockFile = new File("maintenance.lock"); OutputStream out = new FileOutputStream(lockFile); DefaultPropertiesPersister p = new DefaultPropertiesPersister(); p.store(props, out, "Maintenance mode lock file"); - + LOG.debug("Created maintenance lock file."); LOG.info("Application is in maintenance mode now."); } catch (Exception e) { LOG.error("Something went wrong. {}", e.getMessage()); } } else { // The active parameter value is not supported response.put("status", "error"); response.put("message", "Parameter value not supported"); } } catch (JSONException e) { LOG.error("Something went wrong using JSON API. {}", e.getMessage()); } return response.toString(); } - + /** - * Secured endpoint that handles initialization request for the given language + * Secured endpoint that handles initialization request for the given + * language + * * @param lang - * @return true if the initialization is completed without errors, false otherwise + * @return true if the initialization is completed without errors, false + * otherwise */ @RequestMapping(value = "${maintenance.init.uri}", method = RequestMethod.POST, produces = "application/json") - public boolean initialize(@RequestParam("lang") String lang, @Value("${mediawiki.protocol}") String protocol, @Value("${mediawiki.api.url}") String apiUrl){ - String url = protocol + lang + "." + apiUrl; - // Starting a new Process - //Process initializeProcess = new Process(ProcessType.INIT); - - CompletableFuture parallelInsertions = - CompletableFuture.allOf(pageService.addAllPages(lang, url), userService.addAllUsers(url)) - .thenCompose(result -> revisionService.addAllRevisions(lang, url)) - .thenCompose(result -> userService.setAllUsersAuthorship()); - try { - /*boolean result = parallelInsertions.get(); - //saving the result of the process - if (result){ - initializeProcess.setProcessResult(ProcessResult.DONE); - }else{ - initializeProcess.setProcessResult(ProcessResult.ERROR); - } - //metadataDAO.addProcess(initializeProcess); - return result;*/ - return parallelInsertions.get(); + public boolean initialize() { + // Starting a new Process + // Process initializeProcess = new Process(ProcessType.INIT); + + CompletableFuture initFuture = CompletableFuture + .allOf(buildUsersAndPagesFutersList().toArray(new CompletableFuture[langs.size() + 1])) + .thenCompose(result -> CompletableFuture + .allOf(buildRevisionsFuturesList().toArray(new CompletableFuture[langs.size()]))) + .thenCompose(result -> userService.setAllUsersAuthorship()); + + /* + * CompletableFuture parallelInsertions = + * CompletableFuture.allOf(pageService.addAllPages(lang, url), + * userService.addAllUsers(url)) .thenCompose(result -> + * revisionService.addAllRevisions(lang, url)) .thenCompose(result -> + * userService.setAllUsersAuthorship()); + */ + try { + /* + * boolean result = parallelInsertions.get(); //saving the result of + * the process if (result){ + * initializeProcess.setProcessResult(ProcessResult.DONE); }else{ + * initializeProcess.setProcessResult(ProcessResult.ERROR); } + * //metadataDAO.addProcess(initializeProcess); return result; + */ + return initFuture.get(); } catch (InterruptedException | ExecutionException e) { - LOG.error("Something went wrong during database initialization. {}", e.getMessage()); + LOG.error("Something went wrong. {}", e.getMessage()); return false; } } - + /** * */ @RequestMapping(value = "${maintenance.wipe.uri}", method = RequestMethod.DELETE, produces = "application/json") - public void wipeDatabase(){ + public void wipeDatabase() { // TODO } + + /** + * Build a list of CompletableFuture. The elements are the fetches of pages' + * revisions from each domain language. + * + * @return a list of CompletableFuture + */ + private List> buildRevisionsFuturesList() { + List> parallelRevisionsFutures = new ArrayList<>(); + // Add revisions fetch for each domain language + for (String lang : langs) { + String url = protocol + lang + "." + apiUrl; + parallelRevisionsFutures.add(revisionService.addAllRevisions(lang, url)); + } + return parallelRevisionsFutures; + } + + /** + * Build a list of CompletableFuture. The first one is the fetch of the + * users from the first domain in mediawiki.langs list. The rest of the + * elements are the fetches of the pages for each language. This + * implementation assumes that the users are shared among domains. + * + * @return a list of CompletableFuture + */ + private List> buildUsersAndPagesFutersList() { + List> usersAndPagesInsertion = new ArrayList<>(); + // Add users fetch as fist operation + usersAndPagesInsertion.add(userService.addAllUsers(protocol + langs.get(0) + "." + apiUrl)); + // Add pages fetch for each domain language + for (String lang : langs) { + String url = protocol + lang + "." + apiUrl; + usersAndPagesInsertion.add(pageService.addAllPages(lang, url)); + } + return usersAndPagesInsertion; + } } diff --git a/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java b/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java index 8d59f65..8631c87 100644 --- a/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java +++ b/src/main/java/org/wikitolearn/wikirating/repository/PageRepository.java @@ -1,31 +1,27 @@ /** * */ package org.wikitolearn.wikirating.repository; -import java.util.List; - -import org.springframework.data.neo4j.annotation.Query; import org.springframework.data.neo4j.repository.GraphRepository; -import org.springframework.transaction.annotation.Transactional; import org.wikitolearn.wikirating.model.Page; /** * @author aletundo * */ public interface PageRepository extends GraphRepository { /** * * @param title * @return */ Page findByTitle(String title); /** * * @param langPageId * @return */ Page findByLangPageId(String langPageId); } diff --git a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java index c3497ed..78742b1 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/RevisionService.java @@ -1,74 +1,69 @@ /** * */ package org.wikitolearn.wikirating.service; -import java.util.HashSet; import java.util.List; import java.util.concurrent.CompletableFuture; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.model.Page; import org.wikitolearn.wikirating.model.Revision; -import org.wikitolearn.wikirating.model.User; import org.wikitolearn.wikirating.repository.PageRepository; import org.wikitolearn.wikirating.repository.RevisionRepository; -import org.wikitolearn.wikirating.repository.UserRepository; import org.wikitolearn.wikirating.service.mediawiki.RevisionMediaWikiService; /** * * @author aletundo, valsdav * */ @Service public class RevisionService { private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); @Autowired private RevisionMediaWikiService revisionMediaWikiService; @Autowired private RevisionRepository revisionRepository; @Autowired private PageRepository pageRepository; - @Autowired - private UserRepository userRepository; /** * This method inserts all the revisions for every page, creating the * connections between them and between the users that have written them. * * @param lang * String * @param apiUrl * String The MediaWiki API url * @return CompletableFuture */ @Async public CompletableFuture addAllRevisions(String lang, String apiUrl) { Iterable pages = pageRepository.findAll(); pages.forEach(page -> { List revisions = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, page.getPageid()); // Set the first and the last revisions for the current page page.setFistRevision(revisions.get(0)); page.setLastRevision(revisions.get(revisions.size() - 1)); revisions.forEach(revision -> { revision.setLangRevId(lang + "_" + revision.getRevid()); // Get the previous revision filtering the collection for(Revision r : revisions){ if(r.getRevid() == revision.getParentid()){ revision.setPreviousRevision(r); } } }); revisionRepository.save(revisions); pageRepository.save(page); LOG.info("Inserted revisions for page {}", page.getLangPageId()); }); return CompletableFuture.completedFuture(true); } } diff --git a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java index d63537c..1371909 100644 --- a/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java +++ b/src/main/java/org/wikitolearn/wikirating/util/MediaWikiApiUtils.java @@ -1,152 +1,168 @@ /** * */ package org.wikitolearn.wikirating.util; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.UnsupportedEncodingException; import java.util.HashMap; import java.util.Map; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikidata.wdtk.wikibaseapi.LoginFailedException; /** * @author aletundo, valsdav * */ @Component public class MediaWikiApiUtils { private static final Logger LOG = LoggerFactory.getLogger(MediaWikiApiUtils.class); + @Value("${mediawiki.api.user}") + private String apiUser; + @Value("${mediawiki.api.password}") + private String apiPassword; /** * This method creates a MediaWiki Connection object * @param apiUrl the url of the API * @return ApiConnection the API connection object */ public ApiConnection getApiConnection(String apiUrl) { ApiConnection connection = new ApiConnection(apiUrl); - return connection; + if(connection.isLoggedIn()){ + return connection; + }else{ + try { + connection.login(apiUser, apiPassword); + } catch (LoginFailedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + return connection; + } } /** * This method constructs the Map of parameters to attach with the MediaWiki Query to fetch all the pages * in the specified namespace * @param namespace The namespace whose pages are requested * @return Map having parameters */ public Map getListAllPagesParamsMap(String namespace) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allpages"); queryParameterMap.put("aplimit", "max"); queryParameterMap.put("apnamespace", namespace); queryParameterMap.put("apfilterredir", "nonredirects"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to fetch all the revisions * of the given page * @param pid The PageID of the page for which revisions are requested * @return Map having parameters */ public Map getRevisionParam(int pid) { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("prop", "revisions"); queryParameterMap.put("pageids", Integer.toString(pid)); queryParameterMap.put("rvprop", "userid|ids|timestamp|flags|size"); queryParameterMap.put("rvlimit", "max"); queryParameterMap.put("rvdir", "newer"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method constructs the MAP of parameters to attach with the MediaWiki Query to get * all the users. * @return Map having parameters */ public Map getUserParam() { Map queryParameterMap = new HashMap(); queryParameterMap.put("action", "query"); queryParameterMap.put("list", "allusers"); queryParameterMap.put("aulimit", "max"); queryParameterMap.put("format", "json"); return queryParameterMap; } /** * This method converts an InputStream object to String * @param inputStream InputStream object to be converted * @return String result the converted stream into a string */ public JSONObject streamToJson(InputStream inputStream) { String result = ""; BufferedReader reader = null; try { reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); } catch (UnsupportedEncodingException e1) { LOG.error("Unsupported enconding. {}", e1.getMessage()); } StringBuilder builder = new StringBuilder(); String line; try { if(reader!=null){ while ((line = reader.readLine()) != null) { builder.append(line); } } result = builder.toString(); inputStream.close(); } catch (IOException e) { LOG.error("An error occurs while reading the inputStream. {}", e.getMessage()); } JSONObject jsonResponse = null; try { jsonResponse = new JSONObject(result); } catch (JSONException e) { LOG.error("An error occurs while converting string to JSONObject. {}", e.getMessage()); } return jsonResponse; } /** * This method sends a request to MediaWiki API and then gets back an InputStream * @param connection ApiConnection The ApiConnection object * @param requestMethod RequestMethod The request method (ex: GET, POST, ...) * @param queryParametersMap Map The HashMap having all the query parameters * @return response InputStream The result data */ public InputStream sendRequest(ApiConnection connection, String requestMethod, Map queryParametersMap) { InputStream response = null; try { response = connection.sendRequest(requestMethod, queryParametersMap); } catch (IOException e) { LOG.error("Failed to send a request to MediaWiki API. {}", e.getMessage()); } return response; } } diff --git a/src/main/resources/application.example.properties b/src/main/resources/application.example.properties index 1d7600a..4440c52 100644 --- a/src/main/resources/application.example.properties +++ b/src/main/resources/application.example.properties @@ -1,61 +1,63 @@ # ---------------------------------------- # The following properties values are for example purposes and are not intended # to be used in production. # Please, copy the content in application.properties file or where you prefer # and change the values. # ---------------------------------------- # ---------------------------------------- # DATABASE PROPERTIES # ---------------------------------------- spring.data.neo4j.uri=bolt://localhost:7687 spring.data.neo4j.username=neo4j spring.data.neo4j.password=neo4j # ---------------------------------------- # MAINTENANCE PROPERTIES # ---------------------------------------- maintenance.path=/maintenance maintenance.readonlymode.uri=${maintenance.path}/read_only maintenance.init.uri=${maintenance.path}/init maintenance.wipe.uri=${maintenance.path}/wipe # ---------------------------------------- # SECURITY PROPERTIES # ---------------------------------------- security.basic.authorize-mode=role security.basic.realm=Maintenance security.basic.path=${maintenance.path}/** security.user.name=admin security.user.role=ADMIN security.user.password=secret security.headers.xss=true # ---------------------------------------- # MANAGEMENT PROPERTIES # ---------------------------------------- management.port=8081 management.security.roles=ADMIN # ---------------------------------------- # INFO PROPERTIES # ---------------------------------------- info.app.name=WikiRating info.app.description=A Spring Boot application that relies on Neo4j to serve a rating engine for a MediaWiki installation info.app.version=0.0.1 # ---------------------------------------- # ENDPOINT PROPERTIES # ---------------------------------------- endpoints.enabled=false endpoints.info.enabled=true endpoints.health.enabled=true endpoints.metrics.enabled=true endpoints.trace.enabled=true # ---------------------------------------- # MEDIAWIKI PROPERTIES # ---------------------------------------- mediawiki.langs=your,domains,languages,list mediawiki.protocol=http:// or https:// -mediawiki.api.url=your_api_url \ No newline at end of file +mediawiki.api.url=your_api_url +mediawiki.api.user=user +mediawiki.api.password=secret \ No newline at end of file