Huggle  build:^490^dce1e5c
 All Classes Namespaces Functions Variables Enumerations Enumerator Pages
wikiedit.cpp
1 //This program is free software: you can redistribute it and/or modify
2 //it under the terms of the GNU General Public License as published by
3 //the Free Software Foundation, either version 3 of the License, or
4 //(at your option) any later version.
5 
6 //This program is distributed in the hope that it will be useful,
7 //but WITHOUT ANY WARRANTY; without even the implied warranty of
8 //MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 //GNU General Public License for more details.
10 
11 #include "wikiedit.hpp"
12 using namespace Huggle;
13 QList<WikiEdit*> WikiEdit::EditList;
14 
16 {
17  this->RegisterConsumer(HUGGLECONSUMER_WIKIEDIT);
18  this->Bot = false;
19  this->User = NULL;
20  this->Minor = false;
21  this->NewPage = false;
22  this->Size = 0;
23  this->User = NULL;
24  this->Diff = 0;
25  this->OldID = 0;
26  this->Summary = "";
27  this->Status = StatusNone;
28  this->CurrentUserWarningLevel = WarningLevelNone;
29  this->OwnEdit = false;
30  this->EditMadeByHuggle = false;
31  this->TrustworthEdit = false;
32  this->RollbackToken = "";
33  this->PostProcessing = false;
34  this->DifferenceQuery = NULL;
35  this->ProcessingQuery = NULL;
36  this->ProcessingDiff = false;
37  this->ProcessingRevs = false;
38  this->DiffText = "";
39  this->Priority = 20;
40  this->Score = 0;
41  this->Previous = NULL;
42  this->Time = QDateTime::currentDateTime();
43  this->Next = NULL;
44  this->ProcessingByWorkerThread = false;
45  this->ProcessedByWorkerThread = false;
46  this->RevID = WIKI_UNKNOWN_REVID;
47  WikiEdit::EditList.append(this);
48 }
49 
50 WikiEdit::~WikiEdit()
51 {
52  WikiEdit::EditList.removeAll(this);
53  delete this->User;
54  delete this->Page;
55 }
56 
58 {
59  if (this->ProcessedByWorkerThread)
60  {
61  return true;
62  }
63 
64  if (this->ProcessingByWorkerThread)
65  {
66  return false;
67  }
68 
69  if (!this->PostProcessing)
70  {
71  return true;
72  }
73 
74  if (this->ProcessingRevs)
75  {
76  // check if api was processed
77  if (!this->ProcessingQuery->Processed())
78  {
79  return false;
80  }
81 
82  if (this->ProcessingQuery->Result->Failed)
83  {
84  /// \todo LOCALIZE ME
85  Core::Log("Unable to retrieve " + this->User->GetTalk() + " warning level will not be scored by it");
86  } else
87  {
88  // parse the diff now
89  QDomDocument d;
90  d.setContent(this->ProcessingQuery->Result->Data);
91  QDomNodeList page = d.elementsByTagName("rev");
92  QDomNodeList code = d.elementsByTagName("page");
93  bool missing = false;
94  if (code.count() > 0)
95  {
96  QDomElement e = code.at(0).toElement();
97  if (e.attributes().contains("missing"))
98  {
99  missing = true;
100  }
101  }
102  // get last id
103  if (missing != true && page.count() > 0)
104  {
105  QDomElement e = page.at(0).toElement();
106  if (e.nodeName() == "rev")
107  {
108  this->User->SetContentsOfTalkPage(e.text());
109  } else
110  {
111  /// \todo LOCALIZE ME
112  Core::Log("Unable to retrieve " + this->User->GetTalk() + " warning level will not be scored by it");
113  }
114  } else
115  {
116  if (!missing)
117  {
118  /// \todo LOCALIZE ME
119  Core::Log("Unable to retrieve " + this->User->GetTalk() + " warning level will not be scored by it");
120  Core::DebugLog(this->ProcessingQuery->Result->Data);
121  }
122  }
123  }
124  this->ProcessingRevs = false;
125  }
126 
127  if (this->ProcessingDiff)
128  {
129  // check if api was processed
130  if (!this->DifferenceQuery->Processed())
131  {
132  return false;
133  }
134 
135  if (this->DifferenceQuery->Result->Failed)
136  {
137  // whoa it ended in error, we need to get rid of this edit somehow now
138  this->DifferenceQuery->UnregisterConsumer("WikiEdit::PostProcess()");
139  this->DifferenceQuery = NULL;
140  this->PostProcessing = false;
141  return true;
142  }
143 
144  // parse the diff now
145  QDomDocument d;
146  d.setContent(this->DifferenceQuery->Result->Data);
147  QDomNodeList l = d.elementsByTagName("rev");
148  QDomNodeList diff = d.elementsByTagName("diff");
149  // get last id
150  if (l.count() > 0)
151  {
152  QDomElement e = l.at(0).toElement();
153  if (e.nodeName() == "rev")
154  {
155  if (e.text() != "")
156  {
157  this->Page->Contents = e.text();
158  }
159  // check if this revision matches our user
160  if (e.attributes().contains("user"))
161  {
162  if (e.attribute("user") == this->User->Username)
163  {
164  if (e.attributes().contains("rollbacktoken"))
165  {
166  // let's update it from fresh diff
167  this->RollbackToken = e.attribute("rollbacktoken");
168  }
169  }
170  if (e.attributes().contains("revid"))
171  {
172  this->RevID = e.attribute("revid").toInt();
173  }
174  }
175  if (e.attributes().contains("comment"))
176  {
177  this->Summary = e.attribute("comment");
178  }
179  }
180  }
181  if (diff.count() > 0)
182  {
183  QDomElement e = diff.at(0).toElement();
184  if (e.nodeName() == "diff")
185  {
186  this->DiffText = e.text();
187  }
188  } else
189  {
190  Core::DebugLog("Failed to obtain diff for " + this->Page->PageName + " the error was: " + DifferenceQuery->Result->Data);
191  }
192  // we are done processing the diff
193  this->ProcessingDiff = false;
194  }
195 
196  // check if everything was processed and clean up
197  if (this->ProcessingRevs || this->ProcessingDiff)
198  {
199  return false;
200  }
201 
202  if (this->DiffText == "")
203  {
204  /// \todo LOCALIZE ME
205  Core::Log("ERROR: no diff available for " + this->Page->PageName + " unable to rescore");
206  }
207 
208  this->ProcessingQuery->UnregisterConsumer("WikiEdit::PostProcess()");
209  this->ProcessingQuery = NULL;
210  this->DifferenceQuery->UnregisterConsumer("WikiEdit::PostProcess()");
211  this->DifferenceQuery = NULL;
212  this->ProcessingByWorkerThread = true;
213  ProcessorThread::EditLock.lock();
214  ProcessorThread::PendingEdits.append(this);
215  ProcessorThread::EditLock.unlock();
216  return false;
217 }
218 
220 {
221  int xx = 0;
222  /// \todo This whole thing suck we should rewrite it a bit
223  QString text = this->DiffText.toLower();
224  if (this->Page->Contents != "")
225  {
226  text = this->Page->Contents.toLower();
227  }
228  while (xx<Configuration::LocalConfig_ScoreParts.count())
229  {
230  QString w = Configuration::LocalConfig_ScoreParts.at(xx).word;
231  if (text.contains(w))
232  {
233  this->Score += Configuration::LocalConfig_ScoreParts.at(xx).score;
234  ScoreWords.append(w);
235  }
236  xx++;
237  }
238  xx = 0;
239  while (xx<Configuration::LocalConfig_ScoreWords.count())
240  {
241  QString w = Configuration::LocalConfig_ScoreWords.at(xx).word;
242  if (text.contains(" " + w + " ") || text.contains(" " + w + ".")
243  || text.contains(" " + w + ",") || text.contains(" " + w + "!")
244  || text.contains(" " + w + "\n") || text.contains("\n" + w + "\n")
245  || text.contains("\n" + w + " "))
246  {
247  this->Score += Configuration::LocalConfig_ScoreWords.at(xx).score;
248  ScoreWords.append(w);
249  }
250  xx++;
251  }
252 }
253 
255 {
256  if (this->PostProcessing)
257  {
258  return;
259  }
260  this->PostProcessing = true;
261  this->ProcessingQuery = new ApiQuery();
262  this->ProcessingQuery->SetAction(ActionQuery);
263  this->ProcessingQuery->Parameters = "prop=revisions&rvprop=" + QUrl::toPercentEncoding("timestamp|user|comment|content") + "&titles=" +
264  QUrl::toPercentEncoding(this->User->GetTalk());
265  this->ProcessingQuery->RegisterConsumer("WikiEdit::PostProcess()");
266  Core::AppendQuery(this->ProcessingQuery);
267  this->ProcessingQuery->Target = "Retrieving tp " + this->User->GetTalk();
268  this->ProcessingQuery->Process();
269  this->DifferenceQuery = new ApiQuery();
270  this->DifferenceQuery->SetAction(ActionQuery);
271  if (this->RevID != -1)
272  {
273  // &rvprop=content can't be used because of fuck up of mediawiki
274  this->DifferenceQuery->Parameters = "prop=revisions&rvprop=" + QUrl::toPercentEncoding( "ids|user|timestamp|comment" ) + "&rvlimit=1&rvtoken=rollback&rvstartid=" +
275  QString::number(this->RevID) + "&rvdiffto=prev&titles=" +
276  QUrl::toPercentEncoding(this->Page->PageName);
277  } else
278  {
279  this->DifferenceQuery->Parameters = "prop=revisions&rvprop=" + QUrl::toPercentEncoding( "ids|user|timestamp|comment" ) + "&rvlimit=1&rvtoken=rollback&rvdiffto=prev&titles=" +
280  QUrl::toPercentEncoding(this->Page->PageName);
281  }
282  this->DifferenceQuery->Target = Page->PageName;
283  //this->DifferenceQuery->UsingPOST = true;
284  Core::AppendQuery(this->DifferenceQuery);
285  this->DifferenceQuery->RegisterConsumer("WikiEdit::PostProcess()");
286  this->DifferenceQuery->Process();
287  this->ProcessingDiff = true;
288  this->ProcessingRevs = true;
289 }
290 
292 {
293  return Core::GetProjectScriptURL() + "index.php?title=" + QUrl::toPercentEncoding(this->Page->PageName) +
294  "&diff=" + QString::number(this->RevID);
295 }
296 
298 {
299  if (this->Status == StatusPostProcessed)
300  {
301  return true;
302  }
303  return false;
304 }
305 
306 QMutex ProcessorThread::EditLock(QMutex::Recursive);
307 QList<WikiEdit*> ProcessorThread::PendingEdits;
308 
309 void ProcessorThread::run()
310 {
311  while(Core::Running)
312  {
313  ProcessorThread::EditLock.lock();
314  int e=0;
315  while (e<ProcessorThread::PendingEdits.count())
316  {
317  this->Process(PendingEdits.at(e));
318  e++;
319  }
320  PendingEdits.clear();
321  ProcessorThread::EditLock.unlock();
322  QThread::usleep(200000);
323  }
324 }
325 
326 void ProcessorThread::Process(WikiEdit *edit)
327 {
328  // score
329  if (edit->User->IsIP())
330  {
331  edit->Score += Configuration::LocalConfig_IPScore;
332  }
333  if (edit->Bot)
334  {
335  edit->Score += Configuration::LocalConfig_BotScore;
336  }
337  if (edit->Page->IsUserpage())
338  {
339  edit->Score -= 200;
340  }
341  if (edit->Page->IsTalk())
342  {
343  edit->Score -= 2000;
344  }
345  if (edit->Size > 1200 || edit->Size < -1200)
346  {
348  }
349 
350  edit->Score += edit->User->getBadnessScore();
351 
352  edit->ProcessWords();
353 
354  QString TalkPage = edit->User->GetContentsOfTalkPage();
355  if (TalkPage != "")
356  {
357  edit->User->WarningLevel = WikiEdit::GetLevel(TalkPage);
358  }
359 
360  switch(edit->User->WarningLevel)
361  {
362  case 1:
363  edit->Score += 200;
364  edit->CurrentUserWarningLevel = WarningLevel1;
365  break;
366  case 2:
367  edit->Score += 1000;
368  edit->CurrentUserWarningLevel = WarningLevel2;
369  break;
370  case 3:
371  edit->Score += 2000;
372  edit->CurrentUserWarningLevel = WarningLevel3;
373  break;
374  case 4:
375  // people with 4 warnings are so much watched that someone probably revert them
376  // faster than you notice, let's put them lower than unattended vandals
377  edit->Score += 1000;
378  edit->CurrentUserWarningLevel = WarningLevel4;
379  break;
380  }
381 
382  edit->PostProcessing = false;
383  edit->ProcessedByWorkerThread = true;
384  edit->RegisterConsumer("DeletionLock");
385  edit->Status = StatusPostProcessed;
386 }
387 
388 int WikiEdit::GetLevel(QString page)
389 {
391  {
392  // we need to get rid of old warnings now
393  QString orig = page;
394  // first we split the page by sections
395  QStringList sections;
396  int CurrentIndex = 0;
397  while (CurrentIndex < page.length())
398  {
399  if (!page.startsWith("==") && !page.contains("\n=="))
400  {
401  // no sections
402  sections.append(page);
403  break;
404  }
405 
406  // we need to get to start of section now
407  CurrentIndex = 0;
408  if (!page.startsWith("==") && page.contains("\n=="))
409  {
410  page = page.mid(page.indexOf("\n==") + 1);
411  }
412 
413  // get to bottom of it
414  int bottom = 0;
415  if (!page.mid(CurrentIndex).contains("\n=="))
416  {
417  sections.append(page);
418  break;
419  }
420  bottom = page.indexOf("\n==", CurrentIndex);
421 
422  QString section = page.mid(0, bottom);
423  page = page.mid(bottom);
424  sections.append(section);
425  }
426 
427  // now we browse all sections and remove these with no current date
428 
429  CurrentIndex = 0;
430 
431  page = orig;
432 
433  while (CurrentIndex < sections.count())
434  {
435  // we need to find a date in this section
436  if (!sections.at(CurrentIndex).contains("(UTC)"))
437  {
438  // there is none
439  page = page.replace(sections.at(CurrentIndex), "");
440  CurrentIndex++;
441  continue;
442  }
443  QString section = sections.at(CurrentIndex);
444  section = section.mid(0, section.indexOf("(UTC)"));
445  if (section.endsWith(" "))
446  {
447  // we remove trailing white space
448  section = section.mid(0, section.length() - 1);
449  }
450 
451  if (!section.contains(","))
452  {
453  // this is some borked date let's remove it
454  page = page.replace(sections.at(CurrentIndex), "");
455  CurrentIndex++;
456  continue;
457  }
458 
459  QString time = section.mid(section.lastIndexOf(","));
460  if (time.length() < 2)
461  {
462  // what the fuck
463  page = page.replace(sections.at(CurrentIndex), "");
464  CurrentIndex++;
465  continue;
466  }
467 
468  // we remove the comma
469  time = time.mid(2);
470  QDate date = QDate::fromString(time, "d MMMM yyyy");
471  if (!date.isValid())
472  {
473  page = page.replace(sections.at(CurrentIndex), "");
474  CurrentIndex++;
475  continue;
476  } else
477  {
478  // now check if it's at least 1 month old
479  if (QDate::currentDate().addDays(Configuration::LocalConfig_TemplateAge) > date)
480  {
481  // we don't want to parse this thing
482  page = page.replace(sections.at(CurrentIndex), "");
483  CurrentIndex++;
484  continue;
485  }
486  }
487  CurrentIndex++;
488  }
489  }
490 
491  int level = 4;
492  while (level > 0)
493  {
494  int xx=0;
495  while (xx<Configuration::LocalConfig_WarningDefs.count())
496  {
497  QString defs=Configuration::LocalConfig_WarningDefs.at(xx);
498  if (Core::GetKeyFromValue(defs).toInt() == level)
499  {
500  if (page.contains(Core::GetValueFromKey(defs)))
501  {
502  return level;
503  }
504  }
505  xx++;
506  }
507  level--;
508  }
509  return 0;
510 }
QString GetContentsOfTalkPage()
GetContentsOfTalkPage returns a precached content of this users talk page If there is a global instan...
Definition: wikiuser.cpp:191
static QString GetProjectScriptURL()
Return a script url like http://en.wikipedia.org/w/.
Definition: core.cpp:674
int OldID
Old id.
Definition: wikiedit.hpp:103
static void Log(QString Message)
Write text to terminal as well as ring log.
Definition: core.cpp:563
bool NewPage
Edit is a new page.
Definition: wikiedit.hpp:95
QString Target
This is optional property which contains a label of target this query is for.
Definition: apiquery.hpp:98
bool IsTalk()
Return true in case this is a talk page.
Definition: wikipage.cpp:96
int RevID
Revision ID.
Definition: wikiedit.hpp:105
virtual bool Processed()
Returns true in case that query is processed.
Definition: query.cpp:45
bool IsIP()
Returns true in case the current user is IP user.
Definition: wikiuser.cpp:240
void UnregisterConsumer(const int consumer)
This function will remove a string which prevent the object from being removed.
Definition: collectable.cpp:68
void RegisterConsumer(const int consumer)
Registers a consumer.
Definition: collectable.cpp:57
static void AppendQuery(Query *item)
Insert a query to internal list of running queries, so that they can be watched This will insert it t...
Definition: core.cpp:557
QString Username
Username.
Definition: wikiuser.hpp:52
long getBadnessScore(bool _resync=true)
Retrieve a badness score for current user, see WikiUser::BadnessScore for more.
Definition: wikiuser.cpp:271
void Process()
Run.
Definition: apiquery.cpp:138
static bool TrimOldWarnings
This is experimental feature that removes the old templates from talk pages when they are being read...
static QList< WikiEdit * > EditList
This list contains reference to all existing edits in memory.
Definition: wikiedit.hpp:71
int Priority
Priority in queue.
Definition: wikiedit.hpp:101
int WarningLevel
Current warning level of user.
Definition: wikiuser.hpp:54
QString Summary
Summary of edit.
Definition: wikiedit.hpp:110
WarningLevel CurrentUserWarningLevel
Current warning level.
Definition: wikiedit.hpp:108
void ProcessWords()
Definition: wikiedit.cpp:219
void PostProcess()
This function is called by internals of huggle.
Definition: wikiedit.cpp:254
bool EditMadeByHuggle
If this is true the edit was made by huggle.
Definition: wikiedit.hpp:114
bool IsPostProcessed()
Return true in case this edit was post processed already.
Definition: wikiedit.cpp:297
void SetAction(const Action action)
Change the action type.
Definition: apiquery.cpp:185
WikiEdit()
Creates a new empty wiki edit.
Definition: wikiedit.cpp:15
bool Minor
Edit is a minor edit.
Definition: wikiedit.hpp:91
int Diff
Diff id.
Definition: wikiedit.hpp:99
static int GetLevel(QString page)
Get a level of warning from talk page.
Definition: wikiedit.cpp:388
static int LocalConfig_ScoreChange
Score that is added for every edit that has really big size.
WikiUser * User
User who changed the page.
Definition: wikiedit.hpp:89
static bool Running
Change this to false when you want to terminate all threads properly (you will need to wait few ms) ...
Definition: core.hpp:137
static void DebugLog(QString Message, unsigned int Verbosity=1)
This log is only shown if verbosity is same or larger than requested verbosity.
Definition: core.cpp:641
QString GetFullUrl()
Return a full url to edit.
Definition: wikiedit.cpp:291
QString Parameters
Parameters for action, for example page title.
Definition: apiquery.hpp:84
bool FinalizePostProcessing()
This function is called by core.
Definition: wikiedit.cpp:57
int Size
Size of change of edit.
Definition: wikiedit.hpp:97
Wiki edit.
Definition: wikiedit.hpp:67
QString GetTalk()
Return a link to talk page of this user (like User talk:Jimbo)
Definition: wikiuser.cpp:245
QString PageName
Name of page.
Definition: wikipage.hpp:48
This class can be used to execute any kind of api query on any wiki.
Definition: apiquery.hpp:55
bool Bot
Edit is a bot edit.
Definition: wikiedit.hpp:93
bool OwnEdit
Edit was made by you.
Definition: wikiedit.hpp:119
WikiPage * Page
Page that was changed by edit.
Definition: wikiedit.hpp:87
void SetContentsOfTalkPage(QString text)
SetContentsOfTalkPage Change a cache for talk page in local and global cache.
Definition: wikiuser.cpp:212
QString Data
Data retrieved by query.
Definition: queryresult.hpp:25
QueryResult * Result
Result of query, see documentation of QueryResult for more.
Definition: query.hpp:68