diff --git a/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java b/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java index 9c2b081..df5de15 100644 --- a/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java +++ b/src/main/java/org/wikitolearn/wikirating/controller/MaintenanceController.java @@ -1,119 +1,130 @@ /** * */ package org.wikitolearn.wikirating.controller; import java.io.File; import java.io.FileOutputStream; import java.io.OutputStream; import java.time.Instant; import java.util.HashMap; import java.util.Map; import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.HttpStatus; import org.springframework.util.DefaultPropertiesPersister; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.bind.annotation.ResponseStatus; import org.springframework.web.bind.annotation.RestController; +import org.wikitolearn.wikirating.exception.UpdateGraphException; import org.wikitolearn.wikirating.model.api.ApiResponse; import org.wikitolearn.wikirating.model.api.ApiResponseError; import org.wikitolearn.wikirating.model.api.ApiResponseFail; import org.wikitolearn.wikirating.model.api.ApiResponseSuccess; import org.wikitolearn.wikirating.service.MaintenanceService; /** * * @author aletundo * */ @RestController public class MaintenanceController { private static final Logger LOG = LoggerFactory.getLogger(MaintenanceController.class); @Autowired private MaintenanceService maintenanceService; /** * Secured endpoint to enable or disable read only mode. When read only mode * is enabled only GET requests are allowed. To change this behavior @see * org.wikitolearn.wikirating.wikirating.filter.MaintenanceFilter. * * @param active * String requested parameter that toggle the re. Binary values * are accepted. * @return a JSON object with the response */ @RequestMapping(value = "${maintenance.readonlymode.uri}", method = RequestMethod.POST, produces = "application/json") @ResponseBody @ResponseStatus(HttpStatus.ACCEPTED) public ApiResponse toggleReadOnlyMode(@RequestParam(value = "active") String active) { int mode = Integer.parseInt(active); ApiResponse body; if (mode == 0) { // Delete maintenance lock file if it exists File f = new File("maintenance.lock"); if (f.exists()) { f.delete(); LOG.debug("Deleted maintenance lock file."); } LOG.info("Application is live now."); body = new ApiResponseSuccess("Application is live now", Instant.now().toEpochMilli()); } else if (mode == 1) { try { // Create maintenance lock file with a maintenance.active // property set to true Properties props = new Properties(); props.setProperty("maintenance.active", "true"); File lockFile = new File("maintenance.lock"); OutputStream out = new FileOutputStream(lockFile); DefaultPropertiesPersister p = new DefaultPropertiesPersister(); p.store(props, out, "Maintenance mode lock file"); LOG.debug("Created maintenance lock file."); LOG.info("Application is in maintenance mode now."); body = new ApiResponseSuccess("Application is in maintenance mode now", Instant.now().toEpochMilli()); } catch (Exception e) { LOG.error("Something went wrong. {}", e.getMessage()); Map data = new HashMap<>(); data.put("error", HttpStatus.NOT_FOUND.name()); data.put("exception", e.getClass().getCanonicalName()); data.put("stacktrace", e.getStackTrace()); return new ApiResponseError(data, e.getMessage(), HttpStatus.NOT_FOUND.value(), Instant.now().toEpochMilli()); } } else { // The active parameter value is not supported body = new ApiResponseFail("Parameter value not supported", Instant.now().toEpochMilli()); } return body; } /** * Secured endpoint that handles initialization request for the given * language * * @return true if the initialization is completed without errors, false * otherwise */ @RequestMapping(value = "${maintenance.init.uri}", method = RequestMethod.POST, produces = "application/json") public boolean initialize() { return maintenanceService.initializeGraph(); } + @RequestMapping(value = "${maintenance.fetch.uri}", method = RequestMethod.POST, produces = "application/json") + public boolean fetch() { + try { + return maintenanceService.updateGraph(); + }catch (UpdateGraphException e){ + return false; + } + + } + /** * */ @RequestMapping(value = "${maintenance.wipe.uri}", method = RequestMethod.DELETE, produces = "application/json") public void wipeDatabase() { // TODO } } diff --git a/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java b/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java index e8332c6..5e591c9 100644 --- a/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java +++ b/src/main/java/org/wikitolearn/wikirating/service/MaintenanceService.java @@ -1,241 +1,242 @@ package org.wikitolearn.wikirating.service; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.wikitolearn.wikirating.exception.*; import org.wikitolearn.wikirating.model.graph.Process; import org.wikitolearn.wikirating.util.enums.ProcessStatus; import org.wikitolearn.wikirating.util.enums.ProcessType; import java.util.ArrayList; import java.util.Date; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; /** * @author aletundo * @author valsdav */ @Service public class MaintenanceService { private static final Logger LOG = LoggerFactory.getLogger(MaintenanceService.class); @Autowired private UserService userService; @Autowired private PageService pageService; @Autowired private RevisionService revisionService; @Autowired private MetadataService metadataService; @Autowired private ProcessService processService; @Autowired private VoteService voteService; @Value("#{'${mediawiki.langs}'.split(',')}") private List langs; @Value("${mediawiki.protocol}") private String protocol; @Value("${mediawiki.api.url}") private String apiUrl; @Value("${mediawiki.namespace}") private String namespace; /** * Initialize the graph for the first time using parallel threads for each domain language * @return true if initialization succeed */ public boolean initializeGraph() { // Initialize Metadata service metadataService.initMetadata(); // Start a new Process Process initProcess = processService.addProcess(ProcessType.INIT); metadataService.addFirstProcess(initProcess); CompletableFuture initFuture = CompletableFuture // Users and pages .allOf(buildUsersAndPagesFuturesList().toArray(new CompletableFuture[langs.size() + 1])) // CourseStructure .thenCompose(result -> CompletableFuture .allOf(buildApplyCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) // Revisions .thenCompose(result -> CompletableFuture .allOf(buildRevisionsFuturesList().toArray(new CompletableFuture[langs.size()]))) // Change Coefficients .thenCompose(result -> CompletableFuture .allOf(buildChangeCoefficientFuturesList().toArray(new CompletableFuture[langs.size()]))) // Users authorship .thenCompose(result -> userService.initAuthorship()); try { boolean result = initFuture.get(); // Save the result of the process if (result){ processService.closeCurrentProcess(ProcessStatus.DONE); }else{ processService.closeCurrentProcess(ProcessStatus.EXCEPTION); } return result; } catch (InterruptedException | ExecutionException e) { LOG.error("Something went wrong. {}", e.getMessage()); return false; } } /** * Entry point for the scheduled graph updated * @return true if the update succeed */ @Scheduled(cron = "${maintenance.update.cron}") - public void updateGraph() { + public boolean updateGraph() throws UpdateGraphException{ Process currentFetchProcess; Date startTimestampCurrentFetch, startTimestampLatestFetch; // Get start timestamp of the latest FETCH Process before opening a new process startTimestampLatestFetch = (processService.getLastProcessStartDateByType(ProcessType.FETCH) != null) ? processService.getLastProcessStartDateByType(ProcessType.FETCH) : processService.getLastProcessStartDateByType(ProcessType.INIT); // Create a new FETCH process try { currentFetchProcess = processService.addProcess(ProcessType.FETCH); metadataService.updateLatestProcess(); startTimestampCurrentFetch = currentFetchProcess.getStartOfProcess(); } catch (PreviousProcessOngoingException e){ LOG.error("Cannot start Update process because the previous process is still ONGOING." + "The update will be aborted."); - return; + return false; } try { CompletableFuture updateFuture = CompletableFuture .allOf(userService.updateUsers(protocol + langs.get(0) + "." + apiUrl, startTimestampLatestFetch, startTimestampCurrentFetch)) .thenCompose(result -> CompletableFuture .allOf(buildUpdatePagesFuturesList(startTimestampLatestFetch, startTimestampCurrentFetch) .toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> CompletableFuture .allOf(buildApplyCourseStructureFuturesList().toArray(new CompletableFuture[langs.size()]))) .thenCompose(result -> voteService.validateTemporaryVotes(startTimestampCurrentFetch)); boolean result = updateFuture.get(); // Save the result of the process, closing the current one if (result) { processService.closeCurrentProcess(ProcessStatus.DONE); } else { processService.closeCurrentProcess(ProcessStatus.ERROR); } + return result; } catch (TemporaryVoteValidationException | UpdateUsersException | UpdatePagesAndRevisionsException | InterruptedException | ExecutionException e) { processService.closeCurrentProcess(ProcessStatus.EXCEPTION); LOG.error("An error occurred during a scheduled graph update procedure"); throw new UpdateGraphException(); } } /** * Build a list of CompletableFuture. * * @return a list of CompletableFuture */ private List> buildApplyCourseStructureFuturesList() { List> parallelUpdateCourseStructureFutures = new ArrayList<>(); // Add course structure for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelUpdateCourseStructureFutures.add(pageService.applyCourseStructure(lang, url)); } return parallelUpdateCourseStructureFutures; } /** * * @param start * @param end * @return */ private List> buildUpdatePagesFuturesList(Date start, Date end) { List> futures = new ArrayList<>(); // Add update pages for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; futures.add(pageService.updatePages(lang, url, start, end)); } return futures; } /** * Build a list of CompletableFuture. The elements are the fetches of pages' * revisions from each domain language. * * @return a list of CompletableFuture */ private List> buildRevisionsFuturesList() { List> parallelRevisionsFutures = new ArrayList<>(); // Add revisions fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelRevisionsFutures.add(revisionService.initRevisions(lang, url)); } return parallelRevisionsFutures; } private List> buildChangeCoefficientFuturesList(){ List> parallelCCFutures = new ArrayList<>(); // Add revisions fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; parallelCCFutures.add(revisionService.calculateChangeCoefficientAllRevisions(lang, url)); } return parallelCCFutures; } /** * Build a list of CompletableFuture. The first one is the fetch of the * users from the first domain in mediawiki.langs list. The rest of the * elements are the fetches of the pages for each language. This * implementation assumes that the users are shared among domains. * * @return a list of CompletableFuture */ private List> buildUsersAndPagesFuturesList() { List> usersAndPagesInsertion = new ArrayList<>(); // Add users fetch as fist operation usersAndPagesInsertion.add(userService.initUsers(protocol + langs.get(0) + "." + apiUrl)); // Add pages fetch for each domain language for (String lang : langs) { String url = protocol + lang + "." + apiUrl; usersAndPagesInsertion.add(pageService.initPages(lang, url)); } return usersAndPagesInsertion; } /*@SuppressWarnings("unchecked") private List> buildFuturesList(Object obj, String methodPrefix) { List> futures = new ArrayList<>(); for (String lang : langs) { String url = protocol + lang + "." + apiUrl; Method[] methods = obj.getClass().getMethods(); for (int i = 0; i < methods.length; i++) { try { if (methods[i].getName().startsWith(methodPrefix) && methods[i].getParameterCount() == 1) { futures.add((CompletableFuture) methods[i].invoke(obj, url)); } else if (methods[i].getName().startsWith(methodPrefix) && methods[i].getParameterCount() == 2) { futures.add((CompletableFuture) methods[i].invoke(obj, lang, url)); } } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException e) { // TODO Auto-generated catch block e.printStackTrace(); } } } return futures; }*/ } diff --git a/src/main/resources/application.example.properties b/src/main/resources/application.example.properties index 05ad862..0a26114 100644 --- a/src/main/resources/application.example.properties +++ b/src/main/resources/application.example.properties @@ -1,65 +1,66 @@ # ---------------------------------------- # The following properties values are for example purposes and are not intended # to be used in production. # Please, copy the content in application.properties file or where you prefer # and change the values. # ---------------------------------------- # ---------------------------------------- # DATABASE PROPERTIES # ---------------------------------------- spring.data.neo4j.uri=bolt://localhost:7687 spring.data.neo4j.username=neo4j spring.data.neo4j.password=neo4j # ---------------------------------------- # MAINTENANCE PROPERTIES # ---------------------------------------- maintenance.path=/maintenance maintenance.readonlymode.uri=${maintenance.path}/read_only maintenance.init.uri=${maintenance.path}/init +maintenance.fetch.uri=${maintenance.path}/fetch maintenance.wipe.uri=${maintenance.path}/wipe maintenance.update.cron=0 0/30 * * * ? # ---------------------------------------- # SECURITY PROPERTIES # ---------------------------------------- security.basic.authorize-mode=role security.basic.realm=Maintenance security.basic.path=${maintenance.path}/** security.user.name=admin security.user.role=ADMIN security.user.password=secret security.headers.xss=true # ---------------------------------------- # MANAGEMENT PROPERTIES # ---------------------------------------- management.port=8081 management.security.roles=ADMIN # ---------------------------------------- # INFO PROPERTIES # ---------------------------------------- info.app.name=WikiRating info.app.description=A Spring Boot application that relies on Neo4j to serve a rating engine for a MediaWiki installation info.app.version=0.0.1 # ---------------------------------------- # ENDPOINT PROPERTIES # ---------------------------------------- endpoints.enabled=false endpoints.info.enabled=true endpoints.health.enabled=true endpoints.metrics.enabled=true endpoints.trace.enabled=true # ---------------------------------------- # MEDIAWIKI PROPERTIES # ---------------------------------------- mediawiki.langs=your,domains,languages,list mediawiki.protocol=http:// or https:// mediawiki.api.url=your_api_url mediawiki.api.user=user mediawiki.api.password=secret mediawiki.namespace=0 \ No newline at end of file