diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5aca120 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +src/main/resources/application.properties + + +### STS ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans + +### IntelliJ IDEA ### +.idea +*.iws +*.iml +*.ipr + +### NetBeans ### +nbproject/private/ +build/ +nbbuild/ +dist/ +nbdist/ +.nb-gradle/ diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar new file mode 100644 index 0000000..5fd4d50 Binary files /dev/null and b/.mvn/wrapper/maven-wrapper.jar differ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..c954cec --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.3.9/apache-maven-3.3.9-bin.zip diff --git a/Dockerfile b/Dockerfile index 5a06e63..1d5bdfb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,47 @@ ############################################################ # Dockerfile to run an OrientDB (Graph) Container ############################################################ FROM java:8-jdk MAINTAINER OrientDB LTD (info@orientdb.com) +# Override the orientdb download location with e.g.: +# docker build -t mine --build-arg ORIENTDB_DOWNLOAD_SERVER=http://repo1.maven.org/maven2/com/orientechnologies/ . +ARG ORIENTDB_DOWNLOAD_SERVER -ENV ORIENTDB_VERSION 2.1.17 -ENV ORIENTDB_DOWNLOAD_MD5 36663388de171e6fdc7f1cab82be92b8 -ENV ORIENTDB_DOWNLOAD_SHA1 e5ec3e7d79b8c294b6246054437d59ab99d2b152 +ENV ORIENTDB_VERSION 2.2.17 +ENV ORIENTDB_DOWNLOAD_MD5 3be5c561fbee52ca6ba12f1637f206fc +ENV ORIENTDB_DOWNLOAD_SHA1 e43104e7dafb301a232212d2300922a086f7aedf + +ENV ORIENTDB_DOWNLOAD_URL ${ORIENTDB_DOWNLOAD_SERVER:-http://central.maven.org/maven2/com/orientechnologies}/orientdb-community/$ORIENTDB_VERSION/orientdb-community-$ORIENTDB_VERSION.tar.gz + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ +&& rm -rf /var/lib/apt/lists/* #download distribution tar, untar and delete databases RUN mkdir /orientdb && \ - wget "http://central.maven.org/maven2/com/orientechnologies/orientdb-community/$ORIENTDB_VERSION/orientdb-community-$ORIENTDB_VERSION.tar.gz" \ + wget $ORIENTDB_DOWNLOAD_URL \ && echo "$ORIENTDB_DOWNLOAD_MD5 *orientdb-community-$ORIENTDB_VERSION.tar.gz" | md5sum -c - \ && echo "$ORIENTDB_DOWNLOAD_SHA1 *orientdb-community-$ORIENTDB_VERSION.tar.gz" | sha1sum -c - \ - && tar -xvzf orientdb-community-$ORIENTDB_VERSION.tar.gz -C /orientdb --strip-components=1\ + && tar -xvzf orientdb-community-$ORIENTDB_VERSION.tar.gz -C /orientdb --strip-components=1 \ && rm orientdb-community-$ORIENTDB_VERSION.tar.gz \ && rm -rf /orientdb/databases/* ENV PATH /orientdb/bin:$PATH VOLUME ["/orientdb/backup", "/orientdb/databases", "/orientdb/config"] WORKDIR /orientdb #OrientDb binary EXPOSE 2424 #OrientDb http EXPOSE 2480 # Default command start the server -CMD ["server.sh"] \ No newline at end of file +CMD ["server.sh"] + +HEALTHCHECK CMD curl --fail http://localhost:2480/ || exit 1 diff --git a/build_orientdb.sh b/build_orientdb.sh index 04eac7e..5faed4f 100755 --- a/build_orientdb.sh +++ b/build_orientdb.sh @@ -1 +1 @@ -docker build -t wikirating/orientdb . +docker build -t wikitolearn/orientdb . diff --git a/docs/wiki_rating_v.0.3.pdf b/docs/wiki_rating_v.0.3.pdf new file mode 100644 index 0000000..ee22ad8 Binary files /dev/null and b/docs/wiki_rating_v.0.3.pdf differ diff --git a/docs/wiki_rating_v.0.3.tex b/docs/wiki_rating_v.0.3.tex new file mode 100644 index 0000000..3853cf2 --- /dev/null +++ b/docs/wiki_rating_v.0.3.tex @@ -0,0 +1,352 @@ +\documentclass[a4paper,11pt]{article} +\usepackage[utf8]{inputenc} +\usepackage[english]{babel} +\usepackage{amsmath} +\usepackage{graphicx} +\usepackage{geometry} +\usepackage{hyperref} +\pagestyle{headings} + +%opening +\title{\textbf{WikiRating Proposition\\Version 0.3}} +\author{\\Davide Valsecchi $<$valsecchi.davide94@gmail.com$>$ +\\Alessandro Tundo $<$alessandrotundo94@gmail.com$>$ } + +\geometry{a4paper,top=4cm,bottom=4cm,left=3.5cm,% +right=3.5cm,heightrounded,bindingoffset=5mm} + +\newcommand{\wir}{\textbf{\textit{WikiRating }}} +\newcommand{\sg}{$\sigma$ } +\newcommand{\al}{$\alpha$ } +\newcommand{\ph}[1]{$\tilde{V}_#1$ } +\newcommand{\pb}[1]{$\bar{V}_#1$ } +\newcommand{\Pii}{$\Phi$ } +\newcommand{\DPii}{\emph{DF} $\Phi$ } + +\begin{document} +\maketitle +\newpage +\tableofcontents +\newpage + +\section{Introduction} + +The main purpose of this project is the creation of an automated system to evalute the reliability and quality of articles and pages on a wiki-style learning platform. +\paragraph{The Problem} +In wiki-style platform, new contents have to be evaluated by expert users to become reliable and to be shared with the public. That work is fondamental to guarantee the +correctness of the platform's content but could slow down the development of highly +partecipated works with a lot of changes. Moreover finding a lot of experienced editors +in a big community could be really difficult. How can we ensure the quality of contents +and meanwhile make the system automatic? +\paragraph{The Solution} +Our base idea is measuring the trustworthiness of the users to evaluate their writing reliability and their ability to review other's works. Every document is voted by users and, using our estimate of their reliability, we can get an overall rating of it. +\\The complete history of the document is followed to have a precise determination of its quality: every version has a weight related to its importance in the total document. +\\Every user's action implies an adjustment of his rating, trying to measure his real reliability. In a system like this, the activity of the users is also important: more active users are a resource for the platform and have to be prized for their work. + +\newpage +\section{The User} \label{sec:user} +Every user of \wir is identified by a unique ID and by two parameters that measures his performance: the trust (or reliability) coefficient ($\mathbf{\alpha}$) and the activity coefficient ($\theta$). + +In the system also an \emph{Anonymous} user is present wit a fixed and very low reliability. Since the unregistered users cannot write any page on the platform, this user is used only as a reference for anonymous votes. + +\subsection{Trust Coefficient} +This parameter is a number between 0 to 1, we call it $ +\alpha$. The higher it is, the more we can trust the user's judgements and we can consider his writings correct. +We want $\alpha$ to depend on two parameter $a$ and $b$ that can vary between 0 +and 1. + +\begin{description} + \item[$\mathbf{a}$] is linked to the quality of user writing evaluated by other + users' votes ( as we'll describe in section \ref{sec:deltaa}, pag. \pageref{sec:deltaa}). + \item[$\mathbf{b}$] is linked to the measured ability of the user to vote other users' documents. It's is an important parameter because we have to measure the reliability also the users that don't write anything on the platform but act only as readers. +\end{description} + +In the section \ref{sec:change_par} at page \pageref{sec:change_par}, we will describe how $a$ and $b$ are calculated and how they varies during user activity. + +$\alpha$ can be calculated, in the simplest way possible, as a weighted mean of the two parameters: +\begin{equation} +\alpha = \frac{A a + B b}{A +B} +\end{equation} + +\subsection{Activity Coefficient} +This coefficient measures the activity level of a user in the platform. We call this number $\gamma$. If the value is +between 0 and 1 the user is considered little active or inactive. If the value is between +1 to 2 the user is considered regular and if the value is greater than 2 he is regarded as +really active. This coefficient is important in the calculations to modify ths user's trust +coefficient, as we will see later. +The activity coefficient is proportional to the ratio of the number of actions done by +the user over time. Furthermore this coefficient decreases quickly over the time if the user is absent from the platform (See section \ref{sec:deltab}, pag. \pageref{sec:deltab}). + +\subsection{Initial values for the user} +The initial values for the coefficients must be set properly according to our estimation +of the user's initial trust level. We can link it to domains of users' email. For example +accounts that use a recognized institutional domain (like \textit{@unimib.it}) will start +with a trust coefficient higher that account that are registered with a student domain +(like \textit{@campus.unimib.it}). That methods can be used to distinguish teachers from +students and from common users. The initial value of \al could also vary over time, +studying periodically the overall reliability of the users associated with every domain. + +\newpage +\section{The Chain of Changes}\label{sec:chch} +We want now to describe how the \wir system acts on the Chain of Changes (\textit{CC}) +of a wiki page or document. + +The history of a document can be described as a descendant chain. First of all a user +creates a new document, then another user makes changes on the top of the first version and saves it: a new ring of the chain is added. Every time an user works on that document +new subsequent rings are added, multiple branches are not allowed. + +Let's start introducing how the rating system works. + +\subsection{Marks} \label{sec:marks} +Every registered user that visits a page can assign a vote between 1 to 10 +that it is then scaled between 0 to 1 for the calculations. This mark refers to the latest version of the document that is always shown to visitors. + + Every vote is associated with user's trust coefficient that determines the weight of the mark. The global evaluation of \emph{the single version} is calculated as a weighted + average of single votes over the trust coefficients: +\begin{equation} \label{eq:v_medio} +\bar{V} = \frac{\displaystyle{\sum_{i=1}^n \alpha_i \cdot V_i}}{\displaystyle{\sum_{i=1}^n \alpha_i}} +\end{equation} +where $V_i$ are the votes and $\alpha_i$ are trust coefficients of the users. + +So, let's make an example: Alice creates the page and receives some votes, then Bob makes +some changes and a new ring of the chain is created. Obviously new votes on Bob's version +not only evaluate Bob's work but also Alice's old work, depending on the number of Bob's +changes. Instead, votes of Alice's work are not related at all with Bob's subsequent editing +but cannot be thrown away because Bob has not created an entirely new document. We need a +new method to calculate global evaluation for the page and above all a criterion to +establish when a version is considered stable. Let's first explorate the latter. + +\subsection{Trustworthiness of Marks} \label{sec:sigma} +Users are not all equally reliable: for example marks from an ungraduate student must be less influential than marks from a teacher. A calculation of global vote for a page does not ensure us that a document has been checked and evaluated \emph{sufficently}. If a page is evaluated as nine but by five users only, we are not satisfied by the level of judgment. Furthermore our opinion could change depending on the five users' trust coefficient. We need a measure of the \emph{trustworthiness of the marks} from now called $\boldsymbol{\sigma}$. + +In our model, the reliability of the evaluation of a page version is not related only to the votes that it has received , but it is related to the quality and quantity of the votes, i.e. to their distribution and to the trust coefficient of the users. A well-evaluated version has not necessarily an excellent average of votes, but it has been carefully evaluated by a sufficient number of users that agree to a certain extent. + +$\boldsymbol{\sigma}$ is an evaluation of the reliability of the average of marks of a document. That's the goal of \sg parameter. + +The \sg of a user's vote, called $\sigma_i$, is calculated as ($i$ is refered to the i-th users): +\begin{equation} \label{eq:single_sigma} +\sigma_i = \alpha_i^B \cdot \left(1- \Delta V\right)^C \qquad \Delta V = |\bar{V}- V_i| +\end{equation} +In a \emph{single version} the overall \sg value is calculated as: +\begin{equation} \label{eq:partial_sigma} +\sigma= \sum_{i=1}^n \sigma_i = \sum_{i=1}^n \alpha_i^B \cdot \left(1- \Delta V\right)^C +\end{equation} +where $B$ and $C$ are constant that have to be chosen. +Thus, \sg depends on $\alpha$, the trust coefficient of the user, and on the deviation of +his vote from the weithed average of the votes as calculated in the eq. \ref{eq:v_medio}. +If votes are distributed in a large range, their mean is less reliable as a global judgment, so \sg is smaller. Wherease if votes are all close to each other, their mean has more significance and the \emph{trustworthiness of the marks} \sg increases. Obviously +votes from more reliable users are more important than votes from users with a low $\alpha$. +Parameters $B$ and $C$ could be used to make higher \al and $(1-\Delta V)$ more +important than smaller values. + +Now, returning to the problem of the Chain of Changes, what happens if Bob edits the document before $\sigma$ threshold is reached? As we have already said, new votes are more important that old ones, but Alice's work judgements cannot be thrown away. + +\subsection{The Decay Process} \label{sec:decay} +For every version of a page, we +have a method to calculate the average vote (eq. \ref{eq:v_medio}, pag. +\pageref{eq:v_medio}) and the cumulative trustworthiness of marks, \sg (eq. +\ref{eq:partial_sigma}, pag. \pageref{eq:partial_sigma}). Every version added to the document changes it but users votes the page in different point of its history. + +Let's make an example: Alice creates a new document of 1500 characters; Bob finds two spelling errors and creates a new revision, made of 10 characters. Some time later, +Alice's work has been voted by two people only because Bob has worked quickly. Bob's version instead has been voted by a lot of people. \\ In this case considering only Bob's revision's marks (the latest) as valid it's not a problem: the votes are related mainly to Alice's work because she has made the real big part of the writing. Votes given to Alice could almost be added directly. + +If we reverse the situation things change a lot. Imagine that Alice, with her 1500 characters, receives a load of votes. Then, after some time, Bob makes a small correction and receives only a few votes. Can we consider only the lates marks as valid? In this case +it would be wrong because the real judgements has been made by marks of the first +version. After Bob's intervention, new votes are still related to previous version. + +After the first two revisions, Charlie arrives on the page, decides to complete a poor +paragraph of the document and saves a new version with 300 characters. He even receives a lot of valutations. In this case the writing has changed considerably and consequentially +old votes are less significative: they lose importance compared to new votes relatively to the number of modified (added or removed) characters. + +If votes lose importance with the addiction of new rings to the Chain of Changes, also trustworthiness of marks \sg has to be scaled along the revisions to make the system coherent. + +Now we'll descrive how this sort of \textit{\textbf{Decay}} takes part in \wir system. + +\subsubsection{V Decay} \label{sec:v_decay} +The Decay of Votes or \emph{V Decay} must give us a way to calculate the average mark of a +stack of versions of a document given the votes and the number of changes of every ring. +We called this mechanism "decay" just because at every new step of the chain, all previous +calculations have to be scaled by a particular factor to take part in the next computing. +Proceeding this way, step by step, is the best practice to make the computations the less +complex possible with the accumulating of revisions. + +\paragraph{Initial data} Consider now a stack of revisions that begins with a new document \footnote{what to do after a given stable version will be explained later in section \ref{sec:after_stable}, pag. \pageref{sec:after_stable}}. +Every version has received a number $N_i$ of votes by $N_i$ users, everyone with his $ +\alpha_i$ coefficient. Every user can vote only one time every version. Taking individually every version, we calculate it's weighted +\emph{partial} average of marks according to eq. \ref{eq:v_medio}, pag. +\pageref{eq:v_medio}, and we call those partial means \pb{n} ($n$ is related to the n-th version).\\ + + +\paragraph{Decay factor (DF)} +Every version is characterized by an amount of changes: as parameters we have the previous and current length of a page, the number of characters added and removed. +We have to create a \textbf{Decay Factor (DF)} $\mathbf{\Phi}$, function of these parameter to represent the information of how much a page has changed between two versions. + +\paragraph{Total average of vtes} Now, we start calculating a progressive \emph{total average} \ph{i} of marks, starting from the first revision. We can use this simple recursive formula : +\begin{equation} \label{eq:V_hat1} +\tilde{V}_1 = \bar{V}_1 \quad\longrightarrow\quad \tilde{V}_n = \frac{\bar{V}_n + \Phi_n +\cdot \tilde{V}_{n-1}}{1+ \Phi_n} +\end{equation} +For every step, the \emph{total} average depends on the \emph{partial} average of votes for the current revision, but the previos \emph{total} average is not lost. +Actually, the new \emph{total} mark is a weighted mean between the current \emph{partial} mark and the previous \emph{total} mark over the \textbf{Decay Factor (DF)} $\mathbf{\Phi}$, that it's different for every $n$. + + +Now we have a clear view of how votes are scaled over the Chain of Changes, we must look at how \sg parameter scales. + +\subsubsection{$\boldsymbol{\sigma}$ Decay} \label{sec:s_decay} + +As we said in section \ref{sec:sigma}, \sg parameter measures the trustworthiness of marks, so if marks loses importance along the chain, the total sigma \sg calculation should consider all the marks but has to evaluate more newer votes. +\paragraph{Initial data} We know how to calculate \sg for every \emph{single} version from equations \ref{eq:partial_sigma} and \ref{eq:single_sigma} on page \pageref{eq:partial_sigma}. Consider a stack of revisions that begins with a new document. For every version, excepted the current, we can calculate che \emph{partial} \sg and we call that $\sigma_n$. +\begin{equation} \label{eq:sigma_partial_n} +\sigma_{n} = \sum_{i=1}^{N_i} \sigma_i = \sum_{i=1}^{N_i} \alpha_i^B \cdot \left(1- \Delta V\right)^C \qquad \Delta V = |\bar{V_n}- V_i| +\end{equation} +where $N_i$ is the number of votes to the n-th version, $\sigma_i$ and $\alpha_i$ are related to the single vote and user. +First of all, note that for \textit{every version} the overall $\sigma_n$ is calculated using only votes given to that version and that the partial average of marks $\bar{V_n}$ is used, not the total average $\tilde{V_n}$. + +The calculation for past versions can be done only one time because data used doesn't change any more. For the current active version things change a little: in fact every new vote changes the partial average of marks. So, for the latest version $\sigma_n$ must be calculated every time a vote is added. + +\paragraph{Total $\sigma$} Now, as in the case of Votes Decay, we have to calculate the \emph{total} \sg, called $\tilde{\sigma_n}$, starting from the first version. + +\begin{equation} \label{eq:s_hat} +\tilde{\sigma}_1 = \sigma_1 \quad\longrightarrow\quad \tilde{\sigma}_n = \sigma_n + \Phi_n \cdot \tilde{\sigma}_{n-1} +\end{equation} +As Decay Factor (DF) $\Phi$, obviously we use the same DF of the V Decay. + +The effect of the DF is the same of that described in section \ref{sec:v_decay}. The only difference is that $\tilde{\sigma_n}$ is a linear combination of contibutes of every version and not a weighted mean. + + +\newpage +\section{Update User's Parameters} \label{sec:change_par} +As described in section \ref{sec:user}, \al trust coefficient depends on two parameters: the quality of writing $a$ and the qualilty of reviewing $b$. +Both these parameters are changed every time an update process of the \wir database is completed: if the user is the author of one of the revision his $a$ parameter changes, if the user has voted one on more of the revisions his $b$ parameter changes. + +\subsection{$\mathbf{\Delta a}$ calculation} \label{sec:deltaa} +\paragraph{Influencial parameters} +\begin{itemize} +\item In our model, since the \al coeffient is related to the expected rank of the user if evaluated by other users, $\Delta a$ must be proportional to the difference between the user's current \al and the average mark assigned to his work, $\Delta V = ( \bar{V} - \alpha)$. +\item Then $\Delta a$ should depend on the importance of the user's work in the page, thus on the ration $\lambda_n / \mu_{tot}$, where $\lambda_n$ is the number of changes by the user and the $\mu_{tot}$ is the total number of characters of the document at the moment of his revision. +\item Also \sg of votes given to the user could influence the width of $\Delta a$. Note that we are considering the \emph{partial} \sg or $\sigma_n$ of the user's version, as calculated in formula \ref{eq:sigma_partial_n}. If a lot of trusted users vote the document, $\Delta a$ should be more important. To eliminate the influence of the number of voters to this parameter we should use the \emph{reduced} \sg, $\mathring{\sigma}_n$: +\begin{equation} +\mathring{\sigma} _n= \frac{\sigma_n}{N_v} +\end{equation} +where $N_v$ is the number of votes. +\end{itemize} + +When a user writes a revision for a document, every subsequent version is related in some measure with his work. Thus, to evaluate the writing qualities of a user we must consider not only the votes assigned to his version but also all the votes given to subsequent versions till the stable one. Obviously the importance of following versions' votes must be related to the number of changes \emph{from} the first user's version. +We need a kind of \emph{Forward Decay} for $\Delta a$ calculation. + +\paragraph{User's version} First of all let's consider the versions directly written by the current user in exam, Alice. For every Alice's version $n$, $\Delta \tilde{a}_n$ is calculated using the following formula: +\begin{equation} +\Delta \tilde{a}_n = \Delta V_n \cdot \mathring{\sigma_n} \cdot \frac{\lambda_n}{\mu_{tot}} +\end{equation} +using the notation convention of the previous section \ref{sec:decay} we can expand the formula as: +\begin{equation}\label{eq:delta_a_par} +\Delta \tilde{a}_n = \left(\bar{V_n} - \alpha\right) \cdot \frac{\sigma_n}{N_v} \cdot \frac{\lambda_n}{\mu_{tot}} +\end{equation} + +\paragraph{Forward Decay} Now let's consider versions after Alice's work. Starting with the first version after Alice's version we can calculate every \emph{partial} $\Delta \bar{a}_j$ as: +\begin{equation} +\Delta \bar{a}_j = \Psi_j \cdot \Delta V_j \cdot \mathring{\sigma_j} +\end{equation} +\begin{equation} \label{eq:delta_a_DF} +\Delta \bar{a}_j = \Psi_j \cdot \left(\bar{V_j}- \alpha_{Alice}\right) \cdot \frac{\sigma_j}{N_v} +\end{equation} +where $\bar{V_j}$ and $\sigma_j$ are related the j-th version, and $\alpha_{Alice}$ is always the \al coeffient of Alice. + +$\mathbf{\Psi_j}$ is the Decay Factor (DF) that weighs as usually the importance of following versions (instead of $\Phi_n$ that is used with the previous versions). One proposed model for this factor is the inverse of the previous decays factor: +\begin{equation} +\Psi_j = \frac{1}{\Phi_j} +\end{equation} + + +We can observe that if the versions after Alice's one introduces a lot of changes, their votes are not influential for Alice's $\Delta a$. Otherwise the newer votes are also related to Alice's work and consequentally are important for her rank. + +\paragraph{Total $\boldsymbol{\Delta a}$} We can now calculate the total $\Delta a$ for Alice: +\begin{equation} +\Delta a_{tot} = F \cdot \gamma \cdot \left(\sum_{n=1}^A \Delta \tilde{a}_n + \sum_{j=1}^B \Delta \bar{a}_j\right) +\end{equation} +where $F$ is a parameter chosen a posteriori, $A$ is the number of the versions written by Alice and B is the number of versions between Alice's one and the stable one. + +If Alice has written more than once in a single chain of changes: +\begin{itemize} +\item $\Delta \tilde{a}_n$ is calculated for every Alice's version with formula \ref{eq:delta_a_par}. +\item after every Alice's version $\Delta \bar{a}_j$ is calculated with the Forward Decay process with formula \ref{eq:delta_a_DF} for every version till the Decay Factor is below a certain thresold (to improve performances) or another Alice's version is met. +\end{itemize} +As you can see finally $\Delta a_{tot}$ depends on the user's activity coefficient $\gamma$. If the user is inactive ($0 < \gamma < 1$) then $\Delta a_{tot}$ is decreased. If the user is active ($\gamma >1$), then it is increased. An active user could quickly increase his \al if he receives a lot of good votes. + +\subsection{$\mathbf{\Delta b}$ calculation} \label{sec:deltab} +The $b$ parameter measures the user's ability to judge other users' works. This parameter is important because a lot of users on the platform are only viewers and not writers: their \al is fundamental because they are who votes the documents, hence it has to reflect the quality of their actions. + +\paragraph{Influencial parameters} +\begin{itemize} +\item First of all, an important parameter is the deviation of the user's vote from the mean of votes. If the vote is too distant from the mean it could imply that the user is not good at evaluate properly the document. We could define a parameter $\varphi$ calculated as: +\begin{equation} +\varphi = \left(\frac{\delta - \Delta V}{\delta}\right)^G \quad \longrightarrow \quad \Delta b \propto \delta +\end{equation} +where $G$ is a parameter determined a posteriori and $\delta$ is the \emph{standard deviation} of votes assigned to a \emph{single} version: +\begin{equation} +\delta = \sqrt{ \frac{1}{N} \sum_{i=1}^N \left(V_i - \bar{V}\right)^2} +\end{equation} +where $N$ is the number of votes and $\bar{V}$ is the \emph{partial} average of votes of the current version. $\Delta V$ is the absoulute value of the difference between the vote $V_i$ in exam and $\bar{V}$: +\begin{equation} +\Delta V = | V_i - \bar{V}| +\end{equation} +There are different scenarios: +\begin{itemize} +\item if $\Delta V < \delta$, $\varphi$ is always less then 1 and positive. Thus $\Delta b\propto \varphi$ is positive and user's \al increases. +\item if $\delta < \Delta V < 2\delta$, $\varphi$ is negative and less then 1. Thus $b$ decreases by a small amount and \al decreases. +\item if $\Delta V > 2\delta$, $\varphi$ is negative and greater then 1. So $b$ decreases much and so does \al. +\item finally is $\Delta V \simeq \delta$ then $\varphi \simeq 0$. $b$ and \al doesn't change because the vote is distributed normally. +\end{itemize} + +\item Also user's \al is important to measure $\Delta b$. If a trusted user votes far from the average is little probable that he has judged in the wrong way: we trust that user! Variations of $b$ linked with the distribution of votes must be weighted by user's \al in this way: +\begin{equation} +\Delta b \propto \left(1- \alpha\right)^H +\end{equation} +The exponent $H$ could be setted a posteriori. If a user is already trusted his $\Delta b$ changes slowly. +\end{itemize} + +\paragraph{Total $\mathbf{\Delta b}$} +Given a Chain of Changes between two stable versions and a specific user that has partecipated in the poll, we calculate $\Delta b_n$ for each of his vote using the \emph{partial} average of marks $\bar{V_n}$ of every different version: +\begin{equation} +\Delta b_n = \left(1- \alpha\right)^H \cdot \varphi +\end{equation} +\begin{equation} +\Delta b_n = \left(1- \alpha\right)^H \cdot \left(\frac{\delta - \Delta V}{\delta}\right)^G +\end{equation} +Remeber that every user can vote only one time a revision but can vote more than one in Chain of Changes. +Then $\Delta b_n$ are summed up to make \emph{total} $\Delta b$ of a single user: +\begin{equation} +\Delta b_{tot} = L \cdot \gamma \cdot \sum_{i=1}^N \Delta b_i +\end{equation} +where $N$ is the number of the user's vote and $L$ is a parameter to determine a posteriori as usual. Also the $\gamma$ activity coefficient of the user is present as in the $\Delta a_{tot}$ process. Active users' coefficients could change more quickly than inactive users' ones. + +\subsection{$\boldsymbol{\gamma}$ calculation} \label{sec:gamma} +User's activity coefficient $\gamma$ is important because it influences the size of the variation of $a$ and $b$, so it directly influences changes of \al. +The logic behind the calculations is that if a user is active we can change a lot his rating because it is frequently updated. If a user is inactive changes must occur more slowly because we don't know when coefficients will be updated again and it could be not in the short period. +Activity coefficient could be calcaluted as: +\begin{equation} +\gamma = P \cdot \frac{actions}{days} \cdot Q^{-\mbox{\textit{inactive days}}} +\end{equation} +$\gamma$ is proportional to the number of actions by day and to a constant $P$. +Then, for every day \footnote{another fixed period of time could be chosen} of inactivity, $\gamma$ is scaled by an exponential factor with base $Q > 1$. + +\newpage +\section{Pros and Cons, Further development} +\subsection{Pros} +\begin{description} +\item[Expandable] For how it is build, \wir system is only a valid \emph{base} for a more complex and complete rating system. A lot of parameters can be further modeled and a lot of mechanism have to be developed more. +\item[Flexible] The system has a lot of parameters that influence the calculation to be determined a posteriori with tests and simulations. This increases a lot the flexibility of the system. +\item [Scalable] Calculations involved in the system have been thought keeping in mind computational complexity. We try to minimize the number of necessary operations. +\item [Fair] The system tries to evaluate fairly the quality of every user and of every document. The more a user is active, the more he can improve his reputation. Users' engagement is prized and measured. Document's reliability is evaluated thoughtful and we try to discourage bad polling. +\end{description} + +\subsection{Cons} +\begin{description} +\item[Number of users] To work correctly, \wir system needs a lot of users and a lot of votes to every document. Only with a lot of data, marks and their trustworthiness, measured by $\sigma$, make sense. +\item[Review ability] Users' ability to review other works measurament (See section \ref{sec:deltab}, pag. \pageref{sec:deltab}) could be improve a lot. We need to test the current mechanism because it could evaluate normal votes as bad ones if not set correctly. The system is more good at rating users how writes documents than users that vote them. This could be a problem because the majority of users are reviewers. The problem could be solved measuring more precisely the $b$ coefficient. +\end{description} + + + + +\end{document} diff --git a/mvnw b/mvnw new file mode 100755 index 0000000..a1ba1bf --- /dev/null +++ b/mvnw @@ -0,0 +1,233 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # + # Look for the Apple JDKs first to preserve the existing behaviour, and then look + # for the new JDKs provided by Oracle. + # + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/CurrentJDK/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L /System/Library/Java/JavaVirtualMachines/CurrentJDK ] ; then + # + # Apple JDKs + # + export JAVA_HOME=/System/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -L "/Library/Java/JavaVirtualMachines/CurrentJDK" ] ; then + # + # Oracle JDKs + # + export JAVA_HOME=/Library/Java/JavaVirtualMachines/CurrentJDK/Contents/Home + fi + + if [ -z "$JAVA_HOME" ] && [ -x "/usr/libexec/java_home" ]; then + # + # Apple JDKs + # + export JAVA_HOME=`/usr/libexec/java_home` + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Migwn, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` +fi + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + local basedir=$(pwd) + local wdir=$(pwd) + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + wdir=$(cd "$wdir/.."; pwd) + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-$(find_maven_basedir)} +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# Provide a "standardized" way to retrieve the CLI args that will +# work with both Windows and non-Windows executions. +MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" +export MAVEN_CMD_LINE_ARGS + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100644 index 0000000..2b934e8 --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,145 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +set MAVEN_CMD_LINE_ARGS=%* + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" + +set WRAPPER_JAR="".\.mvn\wrapper\maven-wrapper.jar"" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CMD_LINE_ARGS% +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..4faa6ca --- /dev/null +++ b/pom.xml @@ -0,0 +1,90 @@ + + + 4.0.0 + + org.wikitolearn + wiki-rating + 0.0.1-SNAPSHOT + jar + + wiki-rating + SpringBoot microservice to serve rating functionalities + + + org.springframework.boot + spring-boot-starter-parent + 1.5.2.RELEASE + + + + + UTF-8 + UTF-8 + 1.8 + + + + + org.springframework.boot + spring-boot-starter + + + org.springframework.boot + spring-boot-starter-security + + + org.springframework.boot + spring-boot-starter-actuator + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.boot + spring-boot-starter-web + + + org.wikidata.wdtk + wdtk-wikibaseapi + 0.7.0 + + + org.json + json + + + com.orientechnologies + orientdb-core + 2.2.17 + + + com.orientechnologies + orientdb-graphdb + 2.2.17 + + + com.orientechnologies + orientdb-client + 2.2.17 + + + com.tinkerpop.blueprints + blueprints-core + 2.6.0 + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + + + diff --git a/run_orient.sh b/run_orientdb.sh similarity index 76% rename from run_orient.sh rename to run_orientdb.sh index 81a9cff..774253f 100755 --- a/run_orient.sh +++ b/run_orientdb.sh @@ -1,8 +1,11 @@ +docker stop orientdb +docker rm orientdb + docker run -d -p 2424:2424 -p 2480:2480 \ -v orientdb_config:/orientdb/config \ -v orient_database:/orientdb/databases \ -v orient_backup:/orientdb/backup \ -e ORIENTDB_ROOT_PASSWORD=$1 \ --name orientdb \ - wikirating/orientdb:latest + wikitolearn/orientdb:latest diff --git a/src/main/java/org/wikitolearn/WikiRatingApplication.java b/src/main/java/org/wikitolearn/WikiRatingApplication.java new file mode 100644 index 0000000..c439197 --- /dev/null +++ b/src/main/java/org/wikitolearn/WikiRatingApplication.java @@ -0,0 +1,33 @@ +package org.wikitolearn; + +import java.util.concurrent.Executor; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.scheduling.annotation.AsyncConfigurerSupport; +import org.springframework.scheduling.annotation.EnableAsync; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +@SpringBootApplication +@EnableAsync +/** + * + * @author aletundo + * + */ +public class WikiRatingApplication extends AsyncConfigurerSupport{ + + public static void main(String[] args) { + SpringApplication.run(WikiRatingApplication.class, args); + } + + @Override + public Executor getAsyncExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(5); + executor.setMaxPoolSize(10); + executor.setThreadNamePrefix("ServicesTread-"); + executor.initialize(); + return executor; + } +} diff --git a/src/main/java/org/wikitolearn/controllers/InitializerController.java b/src/main/java/org/wikitolearn/controllers/InitializerController.java new file mode 100644 index 0000000..f53bb4a --- /dev/null +++ b/src/main/java/org/wikitolearn/controllers/InitializerController.java @@ -0,0 +1,83 @@ +/** + * + */ +package org.wikitolearn.controllers; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.wikitolearn.dao.PageDAO; +import org.wikitolearn.dao.RevisionDAO; +import org.wikitolearn.dao.UserDAO; +import org.wikitolearn.services.PageService; +import org.wikitolearn.services.RevisionService; +import org.wikitolearn.services.UserService; + +/** + * + * @author aletundo, valsdav + * + */ +@RestController +public class InitializerController { + + private static final Logger LOG = LoggerFactory.getLogger(InitializerController.class); + + @Autowired + private PageService pageService; + @Autowired + private UserService userService; + @Autowired + private RevisionService revisionService; + @Autowired + private PageDAO pageDao; + @Autowired + private UserDAO userDao; + @Autowired + private RevisionDAO revisionDAO; + + private boolean initializedDB = false; + + @RequestMapping(value = "/init", method = RequestMethod.GET, produces = "application/json") + public boolean initialize(@RequestParam("lang") String lang){ + String apiUrl = "https://" + lang + ".wikitolearn.org/api.php"; + + // Initializing the DB schema, only the first time + if (! initializedDB){ + initializeDbClasses(); + initializedDB = true; + } + + CompletableFuture parallelInsertions = + CompletableFuture.allOf(pageService.addAllPages(lang, apiUrl), userService.addAllUsers(apiUrl)) + .thenCompose(s -> revisionService.addAllRevisions(lang, apiUrl)); + + try { + return parallelInsertions.get(); + } catch (InterruptedException | ExecutionException e) { + LOG.error("Something went wrong during database initialization. {}", e.getMessage()); + return false; + } + } + + + /** + * This methods initializes all the DAO Classes, creating the right types on the DB. + * @return void + */ + private void initializeDbClasses(){ + LOG.info("Creating Database classes..."); + pageDao.createDatabaseClass(); + userDao.createDatabaseClass(); + revisionDAO.createDatabaseClass(); + LOG.info("Completed creation of database classes."); + } + +} diff --git a/src/main/java/org/wikitolearn/controllers/MaintenanceController.java b/src/main/java/org/wikitolearn/controllers/MaintenanceController.java new file mode 100644 index 0000000..0959028 --- /dev/null +++ b/src/main/java/org/wikitolearn/controllers/MaintenanceController.java @@ -0,0 +1,81 @@ +/** + * + */ +package org.wikitolearn.controllers; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.util.Properties; + +import org.json.JSONException; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.DefaultPropertiesPersister; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +/** + * + * @author aletundo + * + */ +@RestController +public class MaintenanceController { + + private static final Logger LOG = LoggerFactory.getLogger(MaintenanceController.class); + + /** + * Secured endpoint to enable or disable maintenance mode. When maintenance mode is enabled only GET requests are + * allowed. To change this behavior see org.wikitolearn.filter.MaintenanceFilter class. + * @param active String Requested parameter that indicates the maintenance mode to set. Binary values are accepted. + * @return response String The JSONObject casted to String with the response + */ + @RequestMapping(value = "${maintenance.uri}", method = RequestMethod.POST, produces = "application/json") + public String maintenanceMode(@RequestParam(value = "active") String active) { + JSONObject response = new JSONObject(); + int mode = Integer.parseInt(active); + + try { + if (mode == 0) { + response.put("status", "success"); + response.put("message", "Application is live now."); + // Delete maintenance lock file if it exists + File f = new File("maintenance.lock"); + if (f.exists()) { + f.delete(); + LOG.debug("Deleted maintenance lock file."); + } + LOG.info("Application is live now."); + } else if (mode == 1) { + try { + response.put("status", "success"); + response.put("message", "Application is in maintenance mode now."); + // Create maintenance lock file with a maintenance.active property setted to true + Properties props = new Properties(); + props.setProperty("maintenance.active", "true"); + File lockFile = new File("maintenance.lock"); + OutputStream out = new FileOutputStream(lockFile); + DefaultPropertiesPersister p = new DefaultPropertiesPersister(); + p.store(props, out, "Maintenance mode lock file"); + + LOG.debug("Created maintenance lock file."); + LOG.info("Application is in maintenance mode now."); + } catch (Exception e) { + LOG.error("Something went wrong. {}", e.getMessage()); + } + + } else { + // The active parameter value is not supported + response.put("status", "error"); + response.put("message", "Parameter value not supported"); + } + } catch (JSONException e) { + LOG.error("Something went wrong using JSON API. {}", e.getMessage()); + } + return response.toString(); + } +} diff --git a/src/main/java/org/wikitolearn/controllers/PageController.java b/src/main/java/org/wikitolearn/controllers/PageController.java new file mode 100644 index 0000000..cab3fb3 --- /dev/null +++ b/src/main/java/org/wikitolearn/controllers/PageController.java @@ -0,0 +1,112 @@ +/** + * + */ +package org.wikitolearn.controllers; + +import java.util.ArrayList; +import java.util.List; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.RestController; +import org.wikitolearn.models.Revision; +import org.wikitolearn.models.Vote; + +/** + * @author aletundo RESTController for pages resources. It handles all the + * requests to /pages. + */ +@RestController +@RequestMapping("/pages/") +public class PageController { + private static final Logger LOG = LoggerFactory.getLogger(PageController.class); + + /** + * Handle GET requests on /pages/{pageId} URI. The last revision of the page is returned. + * + * @param pageId + * int The id of the page + * @return response Revision + */ + @RequestMapping(value = "{pageId}", method = RequestMethod.GET, produces = "application/json") + @ResponseBody + public Revision getLastRevisionByPageId(@PathVariable("pageId") int pageId) { + // TODO Get the last available revision for the requested page + return new Revision(); + } + + /** + * Handle GET requests on /pages/{pageId}/revisions URI. All the revision for the requested page are returned. + * + * @param pageId + * int The id of the page + * @return response The list with all the revisions + */ + @RequestMapping(value = "{pageId}/revisions", method = RequestMethod.GET, produces = "application/json") + @ResponseBody + public List getAllPageRevisions(@PathVariable("pageId") int pageId) { + // TODO Get the requested revision + return new ArrayList(); + } + + /** + * Handle GET requests on /pages/{pageId}/revisions/{revId} URI. The requested page revision is returned. + * + * @param pageId + * int The id of the page + * @param revId + * int The id of the revision + * @return response Revision + */ + @RequestMapping(value = "{pageId}/revisions/{revId}", method = RequestMethod.GET, produces = "application/json") + @ResponseBody + public Revision getRevisionById(@PathVariable("pageId") int pageId, @PathVariable("revId") int revId) { + // TODO Get the requested revision + return new Revision(); + } + + /** + * Handle GET requests on /pages/{pageId}/revisions/{revId}/votes URI. All + * the votes for the requested page revision are returned. + * + * @param pageId + * int The id of the page + * @param revId + * int The id of the revision + * @return response Vote + */ + @RequestMapping(value = "{pageId}/revisions/{revId}/votes", method = RequestMethod.GET, produces = "application/json") + @ResponseBody + public List showVotes(@PathVariable("pageId") int pageId, @PathVariable("revId") int revId) { + + // TODO Get votes from PageService/PageDAO + return new ArrayList(); + } + + /** + * Handle POST requests on /pages/{pageId}/revisions/{revId}/votes URI. It + * expects vote and userId as request parameters. + * + * @param pageId + * int The id of the page + * @param revId + * int The id of the revision + * @param vote + * double The value of the vote + * @param userId + * int The id of the user who submit the request + * @return response Vote + */ + @RequestMapping(value = "{pageId}/revisions/{revId}/votes", method = RequestMethod.POST, produces = "application/json") + @ResponseBody + public Vote addVote(@PathVariable("pageId") int pageId, @PathVariable("revId") int revId, + @RequestParam("vote") double vote, @RequestParam("userId") int userId) { + // TODO Insert the vote into the graph + return new Vote(); + } +} diff --git a/src/main/java/org/wikitolearn/dao/GenericDAO.java b/src/main/java/org/wikitolearn/dao/GenericDAO.java new file mode 100644 index 0000000..a5d72a1 --- /dev/null +++ b/src/main/java/org/wikitolearn/dao/GenericDAO.java @@ -0,0 +1,28 @@ +/** + * + */ +package org.wikitolearn.dao; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.wikitolearn.utils.DbConnection; + +/** + * @author aletundo + * + */ +public abstract class GenericDAO { + + protected final Logger LOG = LoggerFactory.getLogger(getClass()); + @Autowired + protected DbConnection connection; + + /** + * This method is used to create the class on the DB. + * It creates an unique index on the id to avoid duplicated. + * @return void + */ + public abstract void createDatabaseClass(); + +} \ No newline at end of file diff --git a/src/main/java/org/wikitolearn/dao/PageDAO.java b/src/main/java/org/wikitolearn/dao/PageDAO.java new file mode 100644 index 0000000..9ca5bbd --- /dev/null +++ b/src/main/java/org/wikitolearn/dao/PageDAO.java @@ -0,0 +1,107 @@ +/** + * + */ +package org.wikitolearn.dao; + +import com.orientechnologies.orient.core.metadata.schema.OClass; +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.orientechnologies.orient.core.sql.OCommandSQL; +import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; +import com.tinkerpop.blueprints.impls.orient.OrientGraph; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertex; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; + +import org.springframework.stereotype.Repository; +import org.wikitolearn.models.Page; +/** + * + * @author aletundo, valsdav + * + */ +@Repository +public class PageDAO extends GenericDAO{ + + /** + * This method is used to create the class on the DB. + * It creates an unique index on pageid to avoid duplicated. + * @return void + */ + @Override + public void createDatabaseClass() { + LOG.info("Creating DB classes for PageDAO..."); + OrientGraphNoTx graph = connection.getGraphNT(); + try{ + OrientVertexType vertex = graph.createVertexType("Page",1); + vertex.createProperty("pageid", OType.INTEGER).setMandatory(true); + vertex.createProperty("lang", OType.STRING).setMandatory(true); + vertex.createIndex("page_lang", OClass.INDEX_TYPE.UNIQUE, "pageid", "lang"); + //now we want to add clusters + graph.command(new OCommandSQL("ALTER CLASS Page ADDCLUSTER Pages_it")).execute(); + graph.command(new OCommandSQL("ALTER CLASS Page ADDCLUSTER Pages_en")).execute(); + //adding the clusters to the class Page + //graph.getRawGraph().getMetadata().getSchema().reload(); + } catch( Exception e ) { + LOG.error("Something went wrong during class creation. {}.", e.getMessage()); + } finally { + graph.shutdown(); + } + } + + + /** + * Insert all the given pages in the database as vertexes. + * If there are duplicates all the insertion is rolled back. + * @param pages List The pages to be inserted + * @param lang String + * @return boolean True if insertion was committed, false otherwise + */ + public Boolean insertPages(List pages, String lang){ + OrientGraphNoTx graph = connection.getGraphNT(); + LOG.info("Starting to insert pages..."); + try{ + for(Page p : pages){ + Map props = new HashMap<>(); + props.put("pageid", p.getPageid()); + props.put( "title", p.getTitle()); + props.put("lang", lang); + props.put("pageRank", p.getPageRank()); + + OrientVertex pageNode = graph.addVertex("class:Page,cluster:Pages_"+lang, props); + LOG.info("Page inserted " + pageNode.toString()); + } + LOG.info("Pages insertion committed"); + return true; + } catch (ORecordDuplicatedException or) { + LOG.error("Page not inserted because it's duplicated. {}", or.getMessage()); + } catch( Exception e ) { + LOG.error("Something went wrong during page insertion. {}", e.getMessage()); + }finally { + graph.shutdown(); + } + return false; + } + + /** + * This methods returns an Iterable over all the pages belonging to a certain cluster, + * so coming from the same language domain. + * @param graph OrientGraph An OrientGraph instance + * @param lang String The language of the cluster + * @return result Iterable with all the pages of the cluster + */ + public Iterable getPagesIteratorFromCluster(OrientGraph graph, String lang){ + Iterable result = null; + try { + result = (Iterable) graph.command(new OCommandSQL( + "SELECT FROM cluster:Pages_"+ lang)).execute(); + } catch (Exception e){ + LOG.error("Something went wrong during quering for pages. {}", e.getMessage()); + } + return result; + } +} diff --git a/src/main/java/org/wikitolearn/dao/RevisionDAO.java b/src/main/java/org/wikitolearn/dao/RevisionDAO.java new file mode 100644 index 0000000..0182659 --- /dev/null +++ b/src/main/java/org/wikitolearn/dao/RevisionDAO.java @@ -0,0 +1,156 @@ +package org.wikitolearn.dao; + +import com.orientechnologies.orient.core.metadata.schema.OClass; +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.orientechnologies.orient.core.sql.OCommandSQL; +import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; +import com.tinkerpop.blueprints.Vertex; +import com.tinkerpop.blueprints.impls.orient.OrientGraph; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertex; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; +import org.springframework.stereotype.Repository; +import org.wikitolearn.models.Revision; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; + +/** + * + * @author aletundo, valsdav + * + */ +@Repository +public class RevisionDAO extends GenericDAO{ + + /** + * This method is used to create the classes on the DB. + * Moreover it creates a unique index on the userid property to avoid duplication. + * @return void + */ + @Override + public void createDatabaseClass() { + LOG.info("Creating DB classes for RevisionDAO..."); + OrientGraphNoTx graph = connection.getGraphNT(); + try{ + // Vertex type for the revision + OrientVertexType vertex = graph.createVertexType("Revision"); + vertex.createProperty("revid", OType.INTEGER).setMandatory(true); + vertex.createProperty("lang", OType.STRING).setMandatory(true); + vertex.createIndex("revid", OClass.INDEX_TYPE.UNIQUE, "revid", "lang"); + // Creating clusters for Revision class + graph.command(new OCommandSQL("ALTER CLASS Revision ADDCLUSTER Revs_it")).execute(); + graph.command(new OCommandSQL("ALTER CLASS Revision ADDCLUSTER Revs_en")).execute(); + // Edge type for the created edge from User to Revision + graph.createEdgeType("Author"); + // Edge type to connect revision to parent revision + graph.createEdgeType("ParentRevision"); + // Edge type to connect last revision to page vertex + graph.createEdgeType("LastRevision"); + // Edge type to connect the first revision of a page + graph.createEdgeType("FirstRevision"); + } catch( Exception e ) { + LOG.error("Something went wrong during class creation. {}.", e.getMessage()); + } finally { + graph.shutdown(); + } + } + + + /** + * This method will insert the revisions of one page, creating the link ParentRevision between them and + * the link FirstRevision and LastRevision with the Page vertex. Moreover it connects the Users to + * the revisions they have created. + * This method must be used only for the first INIT import, NOT for incremental insertion. + * @param pageId + * @param revs + * @return + */ + public Boolean insertRevisions(int pageId, List revs, String lang){ + OrientGraphNoTx graph = connection.getGraphNT(); + LOG.info("Starting to insert revisions..."); + HashMap revsNodes = new HashMap(); + Vertex firstRev = null; + Vertex lastRev = null; + try{ + for(Revision rev : revs){ + Map props = new HashMap<>(); + props.put("revid", rev.getRevid()); + props.put("lang", lang); + props.put("length", rev.getLength()); + props.put("changeCoefficient", rev.getChangeCoefficient()); + props.put( "currentMeanVote", rev.getCurrentMeanVote()); + props.put( "currentVotesReliability", rev.getCurrentVotesReliability()); + props.put( "currentNormalizedVotesReliability", rev.getCurrentNormalisesVotesReliability()); + props.put( "totalMeanVote", rev.getTotalMeanVote()); + props.put( "totalVotesReliability", rev.getTotalVotesReliability()); + props.put( "totalNormalizedVotesReliability", rev.getTotalNormalisesVotesReliability()); + props.put("validated", rev.isValidated()); + + Vertex revNode = graph.addVertex("class:Revision,cluster:Revs_"+lang, props); + //LOG.info("Revision inserted {}.", revNode.toString()); + revsNodes.put(Integer.toString(rev.getRevid()), revNode); + + if (rev.getParentid() == 0){ + firstRev = revNode; + } + if (lastRev==null || rev.getRevid() > (int) lastRev.getProperty("revid")){ + lastRev = revNode; + } + + // Connecting the creator of the revisions + Vertex userCreator = null; + try{ + userCreator = graph.getVertices("User.userid", rev.getUserid()).iterator().next(); + } catch (NoSuchElementException e){ + //if the user is not found we link it to the Anonymous user. + userCreator = graph.getVertices("User.userid", "0" ).iterator().next(); + } + graph.addEdge("class:Author", userCreator, revNode, "Author"); + } + + // Now we have to create the the links between revisions + for (Revision r : revs){ + if (r.getParentid() != 0){ + graph.addEdge("class:ParentRevision", revsNodes.get(Integer.toString(r.getRevid())), + revsNodes.get(Integer.toString(r.getParentid())), "ParentRevision"); + } + } + + // Now let's create the LastRevision and FirstRevision edges + Vertex page = graph.getVertices("Page.pageid", pageId).iterator().next(); + graph.addEdge("class:LastRevision", page, lastRev, "LastRevision"); + graph.addEdge("class:FirstRevision", page, firstRev, "FirstRevision"); + + LOG.info("Revisions of page {} insertion committed", pageId); + return true; + } catch (ORecordDuplicatedException or) { + LOG.error("Some of the pages are duplicates. {}", or.getMessage()); + } catch( Exception e ) { + LOG.error("Something went wrong during user insertion. {}", e.getMessage()); + } finally { + graph.shutdown(); + } + return false; + } + + /** + * This methods returns an Iterable over all the Revisions belonging to a certain cluster, + * so coming from the same language domain. + * @param lang String The language of the cluster + * @return result Iterable with all the revisions of the cluster + */ + public Iterable getRevisionsIteratorFromCluster(OrientGraph graph, String lang){ + Iterable result = null; + try { + result = (Iterable) graph.command(new OCommandSQL( + "SELECT FROM cluster:Revs_"+ lang)).execute(); + } catch (Exception e){ + LOG.error("Something went wrong during quering for revisions. {}", e.getMessage()); + } + return result; + } + +} diff --git a/src/main/java/org/wikitolearn/dao/UserDAO.java b/src/main/java/org/wikitolearn/dao/UserDAO.java new file mode 100644 index 0000000..d099093 --- /dev/null +++ b/src/main/java/org/wikitolearn/dao/UserDAO.java @@ -0,0 +1,87 @@ +/** + * + */ +package org.wikitolearn.dao; + +import com.orientechnologies.orient.core.metadata.schema.OClass; +import com.orientechnologies.orient.core.metadata.schema.OType; +import com.orientechnologies.orient.core.storage.ORecordDuplicatedException; +import com.tinkerpop.blueprints.Vertex; +import com.tinkerpop.blueprints.impls.orient.OrientGraph; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import com.tinkerpop.blueprints.impls.orient.OrientVertexType; +import org.springframework.stereotype.Repository; +import org.wikitolearn.models.User; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * + * @author aletundo, valsdav + * + */ +@Repository +public class UserDAO extends GenericDAO{ + + /** + * This method is used to create the class on the DB. + * Moreover it creates a unique index on the userid property to avoid duplication. + * @return void + */ + @Override + public void createDatabaseClass() { + LOG.info("Creating DB classes for UserDAO..."); + OrientGraphNoTx graph = connection.getGraphNT(); + try{ + OrientVertexType vertex = graph.createVertexType("User"); + vertex.createProperty("userid", OType.INTEGER).setMandatory(true); + vertex.createIndex("userid", OClass.INDEX_TYPE.UNIQUE, "userid"); + } catch( Exception e ) { + LOG.error("Something went wrong during class creation. {}", e.getMessage()); + } finally { + graph.shutdown(); + } + } + + + /** + * Insert all the given users in the database as vertexes. + * If there are duplicates all the insertion is rolled back. + * @param users List The pages to be inserted + * @return boolean True if insertion was committed, false otherwise + */ + public Boolean insertUsers(List users){ + OrientGraph graph = connection.getGraph(); + LOG.info("Starting to insert users..."); + try{ + for(User p : users) { + try { + Map props = new HashMap<>(); + props.put("userid", p.getUserid()); + props.put("username", p.getUsername()); + props.put("votesReliability", p.getVotesReliability()); + props.put("contributesReliability", p.getContributesReliability()); + props.put("totalReliability", p.getTotalReliability()); + + Vertex userNode = graph.addVertex("class:User", props); + graph.commit(); + LOG.info("User inserted {}", userNode.toString()); + } catch (ORecordDuplicatedException or) { + LOG.error("The user is already in the DB. {}. Operation will be rollbacked.", or.getMessage()); + graph.rollback(); + } + } + LOG.info("Users insertion ended"); + graph.shutdown(); + return true; + } catch( Exception e ) { + LOG.error("Something went wrong during user insertion. {}. Operation will be rollbacked.", e.getMessage()); + graph.rollback(); + graph.shutdown(); + } + return false; + } + +} diff --git a/src/main/java/org/wikitolearn/filters/MaintenanceFilter.java b/src/main/java/org/wikitolearn/filters/MaintenanceFilter.java new file mode 100644 index 0000000..f5b8de4 --- /dev/null +++ b/src/main/java/org/wikitolearn/filters/MaintenanceFilter.java @@ -0,0 +1,72 @@ +/** + * + */ +package org.wikitolearn.filters; + +import java.io.File; +import java.io.IOException; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpMethod; +import org.springframework.stereotype.Component; + +/** + * + * @author aletundo + * + */ +@Component +public class MaintenanceFilter implements Filter { + + private @Value("${maintenance.uri}") String MAINTENANCE_URI; + + /* + * (non-Javadoc) + * + * @see javax.servlet.Filter#destroy() + */ + @Override + public void destroy() {} + + /* + * (non-Javadoc) + * + * @see javax.servlet.Filter#doFilter(javax.servlet.ServletRequest, + * javax.servlet.ServletResponse, javax.servlet.FilterChain) + */ + @Override + public void doFilter(ServletRequest servletRequest, ServletResponse servletResponse, FilterChain filterChain) + throws IOException, ServletException { + HttpServletRequest request = (HttpServletRequest) servletRequest; + HttpServletResponse response = (HttpServletResponse) servletResponse; + // Maintenance lock file to be checked + File lockFile = new File("maintenance.lock"); + // Allow any maintenance URI request, otherwise check if maintenance mode is set and allow GET requests only + if(MAINTENANCE_URI.equals(request.getRequestURI())){ + filterChain.doFilter(servletRequest, servletResponse); + }else if(!HttpMethod.GET.matches(request.getMethod()) && lockFile.exists()){ + response.sendError(HttpServletResponse.SC_METHOD_NOT_ALLOWED, + "Application is in maintenance mode. Only GET requests are supported."); + }else{ + filterChain.doFilter(servletRequest, servletResponse); + } + } + + /* + * (non-Javadoc) + * + * @see javax.servlet.Filter#init(javax.servlet.FilterConfig) + */ + @Override + public void init(FilterConfig filterConfig) throws ServletException {} + +} diff --git a/src/main/java/org/wikitolearn/models/Page.java b/src/main/java/org/wikitolearn/models/Page.java new file mode 100644 index 0000000..ebef0bd --- /dev/null +++ b/src/main/java/org/wikitolearn/models/Page.java @@ -0,0 +1,78 @@ +/** + * + */ +package org.wikitolearn.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * + * @author aletundo, valsdav + * + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class Page { + private String title; + private int pageid; + private double pageRank; + + /** + * + */ + public Page() { } + + /** + * @param title + * @param pageid + * @param pageRank + */ + public Page(String title, int pageid, double pageRank) { + this.title = title; + this.pageid = pageid; + this.pageRank = pageRank; + } + /** + * @return the title + */ + public String getTitle() { + return title; + } + /** + * @param title the title to set + */ + public void setTitle(String title) { + this.title = title; + } + /** + * @return the pageid + */ + public int getPageid() { + return pageid; + } + /** + * @param pageid the pageid to set + */ + public void setPageid(int pageid) { + this.pageid = pageid; + } + /** + * @return the pageRank + */ + public double getPageRank() { + return pageRank; + } + /** + * @param pageRank the pageRank to set + */ + public void setPageRank(double pageRank) { + this.pageRank = pageRank; + } + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "Page [title=" + title + ", pageid=" + pageid + ", pageRank=" + pageRank + "]"; + } + +} diff --git a/src/main/java/org/wikitolearn/models/Revision.java b/src/main/java/org/wikitolearn/models/Revision.java new file mode 100644 index 0000000..646acaf --- /dev/null +++ b/src/main/java/org/wikitolearn/models/Revision.java @@ -0,0 +1,140 @@ +package org.wikitolearn.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * This class handles the data of the Revision of a page. + * Created by valsdav on 14/03/17. + */ + +@JsonIgnoreProperties(ignoreUnknown = true) +public class Revision { + private int revid; + private int userid; + private int parentid; + private long length; + private double changeCoefficient; + private double currentMeanVote; + private double currentVotesReliability; + private double currentNormalisesVotesReliability; + private double totalMeanVote; + private double totalVotesReliability; + private double totalNormalisesVotesReliability; + private boolean validated = false; + + public Revision() { } + + public Revision(int revid, int userid, int parentid, long length, double changeCoefficient, + double currentMeanVote, double currentVotesReliability, + double currentNormalisesVotesReliability, double totalMeanVote, + double totalVotesReliability, double totalNormalisesVotesReliability, boolean validated) { + this.revid = revid; + this.userid = userid; + this.parentid = parentid; + this.length = length; + this.changeCoefficient = changeCoefficient; + this.currentMeanVote = currentMeanVote; + this.currentVotesReliability = currentVotesReliability; + this.currentNormalisesVotesReliability = currentNormalisesVotesReliability; + this.totalMeanVote = totalMeanVote; + this.totalVotesReliability = totalVotesReliability; + this.totalNormalisesVotesReliability = totalNormalisesVotesReliability; + this.validated = validated; + } + + public int getRevid() { + return revid; + } + + public void setRevid(int revid) { + this.revid = revid; + } + + public int getUserid() { + return userid; + } + + public void setUserid(int userid) { + this.userid = userid; + } + + public int getParentid() { + return parentid; + } + + public void setParentid(int parentid) { + this.parentid = parentid; + } + + public long getLength() { + return length; + } + + public void setLength(long length) { + this.length = length; + } + + public double getChangeCoefficient() { + return changeCoefficient; + } + + public void setChangeCoefficient(double changeCoefficient) { + this.changeCoefficient = changeCoefficient; + } + + public double getCurrentMeanVote() { + return currentMeanVote; + } + + public void setCurrentMeanVote(double currentMeanVote) { + this.currentMeanVote = currentMeanVote; + } + + public double getCurrentVotesReliability() { + return currentVotesReliability; + } + + public void setCurrentVotesReliability(double currentVotesReliability) { + this.currentVotesReliability = currentVotesReliability; + } + + public double getCurrentNormalisesVotesReliability() { + return currentNormalisesVotesReliability; + } + + public void setCurrentNormalisesVotesReliability(double currentNormalisesVotesReliability) { + this.currentNormalisesVotesReliability = currentNormalisesVotesReliability; + } + + public double getTotalMeanVote() { + return totalMeanVote; + } + + public void setTotalMeanVote(double totalMeanVote) { + this.totalMeanVote = totalMeanVote; + } + + public double getTotalVotesReliability() { + return totalVotesReliability; + } + + public void setTotalVotesReliability(double totalVotesReliability) { + this.totalVotesReliability = totalVotesReliability; + } + + public double getTotalNormalisesVotesReliability() { + return totalNormalisesVotesReliability; + } + + public void setTotalNormalisesVotesReliability(double totalNormalisesVotesReliability) { + this.totalNormalisesVotesReliability = totalNormalisesVotesReliability; + } + + public boolean isValidated() { + return validated; + } + + public void setValidated(boolean validated) { + this.validated = validated; + } +} diff --git a/src/main/java/org/wikitolearn/models/User.java b/src/main/java/org/wikitolearn/models/User.java new file mode 100644 index 0000000..ac42cc0 --- /dev/null +++ b/src/main/java/org/wikitolearn/models/User.java @@ -0,0 +1,123 @@ +/** + * + */ +package org.wikitolearn.models; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author aletundo, valsdav + * + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class User { + @JsonProperty("name") + private String username; + private int userid; + private double votesReliability; + private double contributesReliability; + private double totalReliability; + + /** + * + */ + public User() { + } + + /** + * @param username + * @param userid + * @param votesReliability + * @param contributesReliability + * @param totalReliability + */ + public User(String username, int userid, double votesReliability, double contributesReliability, + double totalReliability) { + this.username = username; + this.userid = userid; + this.votesReliability = votesReliability; + this.contributesReliability = contributesReliability; + this.totalReliability = totalReliability; + } + + /** + * @return the username + */ + public String getUsername() { + return username; + } + + /** + * @param username the username to set + */ + public void setUsername(String username) { + this.username = username; + } + + /** + * @return the userid + */ + public int getUserid() { + return userid; + } + + /** + * @param userid the userid to set + */ + public void setUserId(int userid) { + this.userid = userid; + } + + /** + * @return the votesReliability + */ + public double getVotesReliability() { + return votesReliability; + } + + /** + * @param votesReliability the votesReliability to set + */ + public void setVotesReliability(double votesReliability) { + this.votesReliability = votesReliability; + } + + /** + * @return the contributesReliability + */ + public double getContributesReliability() { + return contributesReliability; + } + + /** + * @param contributesReliability the contributesReliability to set + */ + public void setContributesReliability(double contributesReliability) { + this.contributesReliability = contributesReliability; + } + + /** + * @return the totalReliability + */ + public double getTotalReliability() { + return totalReliability; + } + + /** + * @param totalReliability the totalReliability to set + */ + public void setTotalReliability(double totalReliability) { + this.totalReliability = totalReliability; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "User [username=" + username + ", userid=" + userid + ", votesReliability=" + votesReliability + + ", contributesReliability=" + contributesReliability + ", totalReliability=" + totalReliability + "]"; + } + +} diff --git a/src/main/java/org/wikitolearn/models/Vote.java b/src/main/java/org/wikitolearn/models/Vote.java new file mode 100644 index 0000000..7a5527c --- /dev/null +++ b/src/main/java/org/wikitolearn/models/Vote.java @@ -0,0 +1,66 @@ +/** + * + */ +package org.wikitolearn.models; + +/** + * @author aletundo, valsdav + * + */ +public class Vote { + + private double value; + private double reliability; + + /** + * + */ + public Vote() { + } + + /** + * @param value + * @param reliability + */ + public Vote(double value, double reliability) { + this.value = value; + this.reliability = reliability; + } + + /** + * @return the value + */ + public double getValue() { + return value; + } + + /** + * @param value the value to set + */ + public void setValue(double value) { + this.value = value; + } + + /** + * @return the reliability + */ + public double getReliability() { + return reliability; + } + + /** + * @param reliability the reliability to set + */ + public void setReliability(double reliability) { + this.reliability = reliability; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "Vote [value=" + value + ", reliability=" + reliability + "]"; + } + +} diff --git a/src/main/java/org/wikitolearn/services/PageService.java b/src/main/java/org/wikitolearn/services/PageService.java new file mode 100644 index 0000000..8ff5abb --- /dev/null +++ b/src/main/java/org/wikitolearn/services/PageService.java @@ -0,0 +1,49 @@ +/** + * + */ +package org.wikitolearn.services; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.wikitolearn.dao.PageDAO; +import org.wikitolearn.models.Page; +import org.wikitolearn.services.mediawiki.PageMediaWikiService; + +/** + * + * @author aletundo, valsdav + * + */ +@Service +public class PageService { + private static final Logger LOG = LoggerFactory.getLogger(PageService.class); + @Autowired + private PageMediaWikiService pageMediaWikiService; + @Autowired + private PageDAO pageDao; + + /** + * This methods inserts all the pages inside the DB querying the MediaWiki API. + * @param lang String + * @param apiUrl String The MediaWiki API url + * @return CompletableFuture + */ + @Async + public CompletableFuture addAllPages( String lang, String apiUrl ){ + List pages = pageMediaWikiService.getAll(apiUrl); + LOG.info("Fetched all the pages"); + boolean insertionResultPages = pageDao.insertPages(pages, lang); + if(insertionResultPages){ + LOG.info("Inserted pages"); + }else{ + LOG.error("Something went wrong during pages insertion"); + } + return CompletableFuture.completedFuture(insertionResultPages); + } +} diff --git a/src/main/java/org/wikitolearn/services/RevisionService.java b/src/main/java/org/wikitolearn/services/RevisionService.java new file mode 100644 index 0000000..9ab012e --- /dev/null +++ b/src/main/java/org/wikitolearn/services/RevisionService.java @@ -0,0 +1,74 @@ +/** + * + */ +package org.wikitolearn.services; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.wikitolearn.dao.PageDAO; +import org.wikitolearn.dao.RevisionDAO; +import org.wikitolearn.models.Revision; +import org.wikitolearn.services.mediawiki.RevisionMediaWikiService; +import org.wikitolearn.utils.DbConnection; + +import com.orientechnologies.orient.core.exception.ODatabaseException; +import com.tinkerpop.blueprints.impls.orient.OrientGraph; +import com.tinkerpop.blueprints.impls.orient.OrientVertex; + +/** + * + * @author aletundo, valsdav + * + */ +@Service +public class RevisionService { + private static final Logger LOG = LoggerFactory.getLogger(RevisionService.class); + @Autowired + private RevisionMediaWikiService revisionMediaWikiService; + @Autowired + private RevisionDAO revisionDao; + @Autowired + private PageDAO pageDao; + @Autowired + private DbConnection dbConnection; + + /** + * This method inserts all the revisions for every page, creating the connections between them + * and between the users that have written them. + * @param lang String + * @param apiUrl String The MediaWiki API url + * @return CompletableFuture + */ + @Async + public CompletableFuture addAllRevisions(String lang, String apiUrl){ + OrientGraph graph = dbConnection.getGraph(); + boolean revInsertionResult = false; + try{ + for (OrientVertex page : pageDao.getPagesIteratorFromCluster(graph, lang)) { + int pageId = page.getProperty("pageid"); + LOG.info("Processing page: {}", pageId); + List revs = revisionMediaWikiService.getAllRevisionByPageId(apiUrl, pageId); + revInsertionResult = revisionDao.insertRevisions(pageId, revs, lang); + if(!revInsertionResult){ + LOG.error("Something was wrong during the insertion of the revisions of page {}", pageId); + } + } + } catch ( ODatabaseException e){ + LOG.error("DB Error during insertion of Revisions. {}", e.getMessage()); + graph.rollback(); + } catch (Exception e){ + LOG.error("Something went wrong during Revisions insertion. {}", e.getMessage()); + graph.rollback(); + } finally { + graph.shutdown(); + } + return CompletableFuture.completedFuture(revInsertionResult); + } + +} diff --git a/src/main/java/org/wikitolearn/services/UserService.java b/src/main/java/org/wikitolearn/services/UserService.java new file mode 100644 index 0000000..c51fc00 --- /dev/null +++ b/src/main/java/org/wikitolearn/services/UserService.java @@ -0,0 +1,51 @@ +/** + * + */ +package org.wikitolearn.services; + +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import org.wikitolearn.dao.UserDAO; +import org.wikitolearn.models.User; +import org.wikitolearn.services.mediawiki.UserMediaWikiService; + +/** + * + * @author aletundo, valsdav + * + */ +@Service +public class UserService { + private static final Logger LOG = LoggerFactory.getLogger(UserService.class); + @Autowired + private UserMediaWikiService userMediaWikiService; + @Autowired + private UserDAO userDao; + + /** + * This methods inserts all the users inside the DB querying the MediaWiki API. + * + * @return CompletableFuture + */ + @Async + public CompletableFuture addAllUsers(String apiUrl){ + List users = userMediaWikiService.getAll(apiUrl); + // Adding the Anonymous user + users.add(new User("Anonymous", 0, 0, 0, 0)); + LOG.info("Fetched all the users"); + boolean insertionResultUsers = userDao.insertUsers(users); + if(insertionResultUsers){ + LOG.info("Inserted users"); + }else{ + LOG.error("Something went wrong during users insertion"); + } + return CompletableFuture.completedFuture(insertionResultUsers); + } + +} diff --git a/src/main/java/org/wikitolearn/services/mediawiki/MediaWikiService.java b/src/main/java/org/wikitolearn/services/mediawiki/MediaWikiService.java new file mode 100644 index 0000000..e64be78 --- /dev/null +++ b/src/main/java/org/wikitolearn/services/mediawiki/MediaWikiService.java @@ -0,0 +1,49 @@ +/** + * + */ +package org.wikitolearn.services.mediawiki; + +import java.util.List; + +import org.json.JSONArray; +import org.json.JSONException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.wikitolearn.utils.MediaWikiApiUtils; + +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * @author aletundo + * @param + * + */ +public abstract class MediaWikiService { + + protected final Logger LOG = LoggerFactory.getLogger(getClass()); + + @Autowired + protected MediaWikiApiUtils mediaWikiApiUtils; + @Autowired + protected ObjectMapper mapper; + + public abstract List getAll(String apiUrl); + + /** + * This method is an utility. It concatenates the given JSONArrays into one. + * @param arrays List The arrays to be concatenated + * @return result JSONArray The resulted JSONArray + * @throws JSONException + */ + protected JSONArray concatArrays(List arrays) throws JSONException{ + JSONArray result = new JSONArray(); + for (JSONArray arr : arrays) { + for (int i = 0; i < arr.length(); i++) { + result.put(arr.get(i)); + } + } + return result; + } + +} diff --git a/src/main/java/org/wikitolearn/services/mediawiki/PageMediaWikiService.java b/src/main/java/org/wikitolearn/services/mediawiki/PageMediaWikiService.java new file mode 100644 index 0000000..6766d6e --- /dev/null +++ b/src/main/java/org/wikitolearn/services/mediawiki/PageMediaWikiService.java @@ -0,0 +1,68 @@ +/** + * + */ +package org.wikitolearn.services.mediawiki; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.stereotype.Service; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikitolearn.models.Page; + +import com.fasterxml.jackson.core.type.TypeReference; + +/** + * + * @author aletundo, valsdav + * + */ +@Service +public class PageMediaWikiService extends MediaWikiService{ + + /** + * Get all the pages from a specified namespace of MediaWiki instance through its API. + * @param apiUrl String The MediaWiki API url + * @return pages List A list that contains all the fetched pages + */ + @Override + public List getAll(String apiUrl){ + ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); + Map parameters = mediaWikiApiUtils.getListAllPagesParamsMap("2800"); + InputStream response; + boolean morePages = true; + JSONArray pagesJson = new JSONArray(); + List toBeConcat = new ArrayList<>(); + List pages = new ArrayList<>(); + + try { + while(morePages){ + response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); + JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); + + toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("allpages")); + + if(responseJson.has("continue")){ + String continueFrom = responseJson.getJSONObject("continue").getString("apcontinue"); + parameters.put("apfrom", continueFrom); + }else{ + morePages = false; + pagesJson = concatArrays(toBeConcat); + } + } + pages = mapper.readValue(pagesJson.toString(), new TypeReference>(){}); + return pages; + } catch (JSONException e){ + LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); + } catch(IOException e){ + LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); + } + return pages; + } +} diff --git a/src/main/java/org/wikitolearn/services/mediawiki/RevisionMediaWikiService.java b/src/main/java/org/wikitolearn/services/mediawiki/RevisionMediaWikiService.java new file mode 100644 index 0000000..0e75e40 --- /dev/null +++ b/src/main/java/org/wikitolearn/services/mediawiki/RevisionMediaWikiService.java @@ -0,0 +1,78 @@ +package org.wikitolearn.services.mediawiki; + +import com.fasterxml.jackson.core.type.TypeReference; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.stereotype.Service; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikitolearn.models.Revision; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * + * @author aletundo, valsdav + * + */ +@Service +public class RevisionMediaWikiService extends MediaWikiService{ + + private Map parameters; + + /** + * Get all the revisions of a page. + * @param apiUrl String The MediaWiki API url + * @return revisions List A list that contains all the fetched revisions + */ + @Override + public List getAll(String apiUrl){ + ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); + InputStream response; + boolean moreRevs = true; + JSONArray revsJson = new JSONArray(); + List toBeConcat = new ArrayList<>(); + List revs = new ArrayList<>(); + + try { + while(moreRevs){ + response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); + JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); + + toBeConcat.add(responseJson.getJSONObject("query").getJSONObject("pages"). + getJSONObject(parameters.get("pageids")).getJSONArray("revisions")); + + if(responseJson.has("continue")){ + String continueFrom = responseJson.getJSONObject("continue").getString("rvcontinue"); + parameters.put("rvcontinue", continueFrom); + }else{ + moreRevs = false; + revsJson = concatArrays(toBeConcat); + } + } + revs = mapper.readValue(revsJson.toString(), new TypeReference>(){}); + return revs; + } catch (JSONException e){ + LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); + } catch(IOException e){ + LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); + } + return revs; + } + + /** + * Get all the revisions for a specific page querying MediaWiki API + * @param apiUrl String The MediaWiki API url + * @param pageid int The id the page of which getting the revisions + * @return revisions List A list that contains all the fetched revisions + */ + public List getAllRevisionByPageId(String apiUrl, int pageId) { + parameters = mediaWikiApiUtils.getRevisionParam(pageId); + return getAll(apiUrl); + } + +} diff --git a/src/main/java/org/wikitolearn/services/mediawiki/UserMediaWikiService.java b/src/main/java/org/wikitolearn/services/mediawiki/UserMediaWikiService.java new file mode 100644 index 0000000..a8a7d33 --- /dev/null +++ b/src/main/java/org/wikitolearn/services/mediawiki/UserMediaWikiService.java @@ -0,0 +1,64 @@ +package org.wikitolearn.services.mediawiki; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.springframework.stereotype.Service; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; +import org.wikitolearn.models.User; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * + * @author aletundo, valsdav + * + */ +@Service +public class UserMediaWikiService extends MediaWikiService{ + /** + * Get all the users from MediaWiki instance through its API. + * @param apiUrl String The MediaWiki API url + * @return users List A list that contains all the fetched users + */ + @Override + public List getAll(String apiUrl){ + ApiConnection connection = mediaWikiApiUtils.getApiConnection(apiUrl); + Map parameters = mediaWikiApiUtils.getUserParam(); + InputStream response; + boolean moreUsers = true; + JSONArray usersJson = new JSONArray(); + List toBeConcat = new ArrayList<>(); + List users = new ArrayList<>(); + + try { + while(moreUsers){ + response = mediaWikiApiUtils.sendRequest(connection, "GET", parameters); + JSONObject responseJson = mediaWikiApiUtils.streamToJson(response); + + toBeConcat.add(responseJson.getJSONObject("query").getJSONArray("allusers")); + + if(responseJson.has("continue")){ + String continueFrom = responseJson.getJSONObject("continue").getString("aufrom"); + parameters.put("aufrom", continueFrom); + }else{ + moreUsers = false; + usersJson = concatArrays(toBeConcat); + } + } + users = mapper.readValue(usersJson.toString(), new TypeReference>(){}); + return users; + } catch (JSONException e){ + LOG.error("An error occurred while a JSONObject or JSONArray. {}", e.getMessage()); + } catch(IOException e){ + LOG.error("An error occurred while converting an InputStream to JSONObject. {}", e.getMessage()); + } + return users; + } +} + diff --git a/src/main/java/org/wikitolearn/utils/DbConnection.java b/src/main/java/org/wikitolearn/utils/DbConnection.java new file mode 100644 index 0000000..d1bb68b --- /dev/null +++ b/src/main/java/org/wikitolearn/utils/DbConnection.java @@ -0,0 +1,53 @@ +/** + * + */ +package org.wikitolearn.utils; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import com.orientechnologies.orient.core.intent.OIntentMassiveInsert; +import com.tinkerpop.blueprints.impls.orient.OrientGraph; +import com.tinkerpop.blueprints.impls.orient.OrientGraphFactory; +import com.tinkerpop.blueprints.impls.orient.OrientGraphNoTx; +import org.wikitolearn.dao.UserDAO; +/** + * @author aletundo, valsdav + * + */ +@Component +public class DbConnection { + + private static final Logger LOG = LoggerFactory.getLogger(UserDAO.class); + private OrientGraphFactory factory; + + @Autowired + public DbConnection(@Value("${db.url}") String dbUrl, @Value("${db.user}") + String dbUser, @Value("${db.password}") String dbPwd){ + + factory = new OrientGraphFactory(dbUrl, dbUser, dbPwd).setupPool(1, 20); + } + /** + * This method will return the OreintDB graph instance after connection. + * @return Transaction enabled OrientGraph object + */ + public OrientGraph getGraph(){ + LOG.info("Getting an instance of OrientDB...."); + return factory.getTx(); + } + + /** + * This method will return the OreintDB graph instance after connection, + * for massive inserts to improve performance,no transaction method + * @return Transaction disabled OrientGraph object + */ + + public OrientGraphNoTx getGraphNT() { + LOG.info("Getting a NoTX instance of OrientDB...."); + factory.declareIntent(new OIntentMassiveInsert()); + return factory.getNoTx(); + } +} diff --git a/src/main/java/org/wikitolearn/utils/MediaWikiApiUtils.java b/src/main/java/org/wikitolearn/utils/MediaWikiApiUtils.java new file mode 100644 index 0000000..ba4e2ed --- /dev/null +++ b/src/main/java/org/wikitolearn/utils/MediaWikiApiUtils.java @@ -0,0 +1,152 @@ +/** + * + */ +package org.wikitolearn.utils; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Map; + +import org.json.JSONException; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; + +/** + * @author aletundo, valsdav + * + */ +@Component +public class MediaWikiApiUtils { + + private static final Logger LOG = LoggerFactory.getLogger(MediaWikiApiUtils.class); + + /** + * This method creates a MediaWiki Connection object + * @param apiUrl the url of the API + * @return ApiConnection the API connection object + */ + + public ApiConnection getApiConnection(String apiUrl) { + ApiConnection connection = new ApiConnection(apiUrl); + return connection; + } + + /** + * This method constructs the Map of parameters to attach with the MediaWiki Query to fetch all the pages + * in the specified namespace + * @param namespace The namespace whose pages are requested + * @return Map having parameters + */ + + public Map getListAllPagesParamsMap(String namespace) { + Map queryParameterMap = new HashMap(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("list", "allpages"); + queryParameterMap.put("aplimit", "max"); + queryParameterMap.put("apnamespace", namespace); + queryParameterMap.put("apfilterredir", "nonredirects"); + queryParameterMap.put("format", "json"); + return queryParameterMap; + } + + /** + * This method constructs the MAP of parameters to attach with the MediaWiki Query to fetch all the revisions + * of the given page + * @param pid The PageID of the page for which revisions are requested + * @return Map having parameters + */ + public Map getRevisionParam(int pid) { + Map queryParameterMap = new HashMap(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("prop", "revisions"); + queryParameterMap.put("pageids", Integer.toString(pid)); + queryParameterMap.put("rvprop", "userid|ids|timestamp|flags|size"); + queryParameterMap.put("rvlimit", "max"); + queryParameterMap.put("rvdir", "newer"); + queryParameterMap.put("format", "json"); + return queryParameterMap; + } + + /** + * This method constructs the MAP of parameters to attach with the MediaWiki Query to get + * all the users. + * @return Map having parameters + */ + public Map getUserParam() { + Map queryParameterMap = new HashMap(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("list", "allusers"); + queryParameterMap.put("aulimit", "max"); + queryParameterMap.put("format", "json"); + return queryParameterMap; + } + + /** + * This method converts an InputStream object to String + * @param inputStream InputStream object to be converted + * @return String result the converted stream into a string + */ + public JSONObject streamToJson(InputStream inputStream) { + + String result = ""; + BufferedReader reader = null; + try { + reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8")); + } catch (UnsupportedEncodingException e1) { + LOG.error("Unsupported enconding. {}", e1.getMessage()); + } + + StringBuilder builder = new StringBuilder(); + String line; + try { + + if(reader!=null){ + + while ((line = reader.readLine()) != null) { + builder.append(line); + } + } + result = builder.toString(); + inputStream.close(); + } catch (IOException e) { + + LOG.error("An error occurs while reading the inputStream. {}", e.getMessage()); + } + + JSONObject jsonResponse = null; + try { + jsonResponse = new JSONObject(result); + } catch (JSONException e) { + LOG.error("An error occurs while converting string to JSONObject. {}", e.getMessage()); + } + + return jsonResponse; + } + + /** + * This method sends a request to MediaWiki API and then gets back an InputStream + * @param connection ApiConnection The ApiConnection object + * @param requestMethod RequestMethod The request method (ex: GET, POST, ...) + * @param queryParametersMap Map The HashMap having all the query parameters + * @return response InputStream The result data + */ + + public InputStream sendRequest(ApiConnection connection, String requestMethod, Map queryParametersMap) { + InputStream response = null; + try { + response = connection.sendRequest(requestMethod, queryParametersMap); + } catch (IOException e) { + + LOG.error("Failed to send a request to MediaWiki API. {}", e.getMessage()); + } + + return response; + } +} diff --git a/src/main/resources/application.example.properties b/src/main/resources/application.example.properties new file mode 100644 index 0000000..5d55443 --- /dev/null +++ b/src/main/resources/application.example.properties @@ -0,0 +1,50 @@ +# ---------------------------------------- +# The following properties values are for example purposes and are not intended +# to be used in production. +# Please, copy the content in application.properties file or where you prefer +# and change the values. +# ---------------------------------------- +# ---------------------------------------- +# DATABASE PROPERTIES +# ---------------------------------------- +db.url=remote:localhost/database +db.user=root +db.password=secret + +# ---------------------------------------- +# MAINTENANCE PROPERTIES +# ---------------------------------------- +maintenance.uri=/admin/maintenance + +# ---------------------------------------- +# SECURITY PROPERTIES +# ---------------------------------------- +security.basic.authorize-mode=role +security.basic.realm=Administration +security.basic.path=/admin/** +security.user.name=admin +security.user.role=ADMIN +security.user.password=secret +security.headers.xss=true + +# ---------------------------------------- +# MANAGEMENT PROPERTIES +# ---------------------------------------- +management.port=8081 +management.security.roles=ADMIN + +# ---------------------------------------- +# INFO PROPERTIES +# ---------------------------------------- +info.app.name=WikiRating +info.app.description=A Spring Boot application that relies on a OrientDb instance to serve a rating engine for a MediaWiki installation +info.app.version=0.0.1 + +# ---------------------------------------- +# ENDPOINT PROPERTIES +# ---------------------------------------- +endpoints.enabled=false +endpoints.info.enabled=true +endpoints.health.enabled=true +endpoints.metrics.enabled=true +endpoints.trace.enabled=true \ No newline at end of file diff --git a/src/test/java/org/wikitolearn/WikiRatingApplicationTests.java b/src/test/java/org/wikitolearn/WikiRatingApplicationTests.java new file mode 100644 index 0000000..836fc8a --- /dev/null +++ b/src/test/java/org/wikitolearn/WikiRatingApplicationTests.java @@ -0,0 +1,16 @@ +package org.wikitolearn; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.junit4.SpringRunner; + +@RunWith(SpringRunner.class) +@SpringBootTest +public class WikiRatingApplicationTests { + + @Test + public void contextLoads() { + } + +} diff --git a/src/test/java/org/wikitolearn/utils/MediaWikiApiUtilsTest.java b/src/test/java/org/wikitolearn/utils/MediaWikiApiUtilsTest.java new file mode 100644 index 0000000..a9d0ae4 --- /dev/null +++ b/src/test/java/org/wikitolearn/utils/MediaWikiApiUtilsTest.java @@ -0,0 +1,120 @@ +/** + * + */ +package org.wikitolearn.utils; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; + +import org.json.JSONObject; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.springframework.test.context.junit4.SpringRunner; +import org.wikidata.wdtk.wikibaseapi.ApiConnection; + +/** + * @author aletundo + * + */ +@RunWith(SpringRunner.class) +public class MediaWikiApiUtilsTest { + + @InjectMocks + private MediaWikiApiUtils mediaWikiApiUtils; + + @Test + public void testGetListAllPagesParamsMap(){ + Map queryParameterMap = new HashMap(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("list", "allpages"); + queryParameterMap.put("aplimit", "max"); + queryParameterMap.put("apnamespace", "2800"); + queryParameterMap.put("apfilterredir", "nonredirects"); + queryParameterMap.put("format", "json"); + + Map result = mediaWikiApiUtils.getListAllPagesParamsMap("2800"); + assertEquals(queryParameterMap.get("action"), result.get("action")); + assertEquals(queryParameterMap.get("list"), result.get("list")); + assertEquals(queryParameterMap.get("aplimit"), result.get("aplimit")); + assertEquals(queryParameterMap.get("apnamespace"), result.get("apnamespace")); + assertEquals(queryParameterMap.get("apfiltereedir"), result.get("apfiltereedir")); + assertEquals(queryParameterMap.get("format"), result.get("format")); + } + + @Test + public void testGetRevisionParam(){ + Map queryParameterMap = new HashMap(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("prop", "revisions"); + queryParameterMap.put("pageids", "1"); + queryParameterMap.put("rvprop", "userid|ids|timestamp|flags|size"); + queryParameterMap.put("rvlimit", "max"); + queryParameterMap.put("rvdir", "newer"); + queryParameterMap.put("format", "json"); + + Map result = mediaWikiApiUtils.getRevisionParam(1); + assertEquals(queryParameterMap.get("action"), result.get("action")); + assertEquals(queryParameterMap.get("prop"), result.get("prop")); + assertEquals(queryParameterMap.get("pageids"), result.get("pageids")); + assertEquals(queryParameterMap.get("rvprop"), result.get("rvprop")); + assertEquals(queryParameterMap.get("rvlimit"), result.get("rvlimit")); + assertEquals(queryParameterMap.get("rvdir"), result.get("rvdir")); + assertEquals(queryParameterMap.get("format"), result.get("format")); + } + + @Test + public void testGetUserparam(){ + Map queryParameterMap = new HashMap(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("list", "allusers"); + queryParameterMap.put("aulimit", "max"); + queryParameterMap.put("format", "json"); + + Map result = mediaWikiApiUtils.getUserParam(); + assertEquals(queryParameterMap.get("action"), result.get("action")); + assertEquals(queryParameterMap.get("list"), result.get("list")); + assertEquals(queryParameterMap.get("aulimit"), result.get("aulimit")); + assertEquals(queryParameterMap.get("format"), result.get("format")); + } + + @Test + public void testGetApiConnection(){ + ApiConnection connection = mediaWikiApiUtils.getApiConnection("https://en.wikitolearn.org/api.php"); + assertTrue(connection instanceof ApiConnection); + assertFalse(connection.isLoggedIn()); + } + + @Test + public void testSendRequest(){ + ApiConnection connection = mediaWikiApiUtils.getApiConnection("https://fake.api"); + ApiConnection connection2 = mediaWikiApiUtils.getApiConnection("https://en.wikitolearn.org/api.php"); + Map queryParameterMap = new HashMap(); + queryParameterMap.put("action", "query"); + queryParameterMap.put("list", "allusers"); + queryParameterMap.put("aulimit", "max"); + queryParameterMap.put("format", "json"); + assertNull(mediaWikiApiUtils.sendRequest(connection, "GET", queryParameterMap)); + assertNotNull(mediaWikiApiUtils.sendRequest(connection2, "GET", queryParameterMap)); + } + + @Test + public void testStreamToJson(){ + String JSONString = "{\"id\" : \"1\"}"; + String JSONStringMalformed = "{\"id : \"1\"}"; + InputStream inputStream = new ByteArrayInputStream(JSONString.getBytes()); + InputStream inputStream2 = new ByteArrayInputStream(JSONStringMalformed.getBytes()); + JSONObject result = mediaWikiApiUtils.streamToJson(inputStream); + JSONObject result2 = mediaWikiApiUtils.streamToJson(inputStream2); + assertNotNull(result); + assertNull(result2); + } +}