9 def __init__(self, name):
11 self.reward_kind = None
14 def __init__(self, name):
21 def is_upgrade_for(self, other):
22 aps = self.amount/self.duration
23 other_aps = other.amount/other.duration
24 return (aps>other_aps and self.amount>other.amount)
27 def __init__(self, name):
29 self.primary_attribute = None
30 self.secondary_attribute = None
33 self.drop_only = False
35 class WikiParser(html.parser.HTMLParser):
37 super(WikiParser, self).__init__()
41 def handle_starttag(self, tag, attrs):
47 if n=="class" and "c-item-hoverbox__display" in v:
50 def handle_endtag(self, tag):
55 class WikiTableParser(WikiParser):
57 super(WikiTableParser, self).__init__()
59 self.in_items_table = False
63 self.current_item = None
65 def handle_starttag(self, tag, attrs):
66 super(WikiTableParser, self).handle_starttag(tag, attrs)
70 if n=="class" and "wikitable" in v:
71 self.in_items_table = True
74 self.current_item = None
77 if self.in_items_table:
80 def handle_endtag(self, tag):
81 super(WikiTableParser, self).handle_endtag(tag)
84 self.in_items_table = False
88 def handle_data(self, data):
98 self.current_item = self.create_item(data)
100 self.items.append(self.current_item)
101 elif self.current_item:
102 self.handle_value(self.column, data)
104 def create_item(self, name):
107 def handle_value(self, column, data):
110 class DivinationCardsParser(WikiTableParser):
111 def handle_starttag(self, tag, attrs):
112 super(DivinationCardsParser, self).handle_starttag(tag, attrs)
114 if tag=="span" and not self.ignore:
115 if self.in_cell and self.current_item and not self.current_item.reward_kind and self.column==3:
119 self.current_item.reward_kind = "currency"
121 self.current_item.reward_kind = "unique"
123 self.current_item.reward_kind = "rare"
124 elif "-magic" in v or "-mod" in v:
125 self.current_item.reward_kind = "magic"
127 self.current_item.reward_kind = "skillgem"
129 def handle_endtag(self, tag):
130 super(DivinationCardsParser, self).handle_endtag(tag)
133 if self.current_item and not self.current_item.reward_kind:
134 self.current_item.reward_kind = "other"
136 def create_item(self, name):
139 def handle_value(self, column, data):
142 self.current_item.reward_kind = "links"
144 self.current_item.reward_kind = "map"
146 class FlasksParser(WikiTableParser):
147 def __init__(self, utility=False):
148 super(FlasksParser, self).__init__()
150 self.utility = utility
152 def handle_endtag(self, tag):
153 super(FlasksParser, self).handle_endtag(tag)
156 if self.current_item:
157 for it in self.items:
158 if not it.upgrade and self.current_item.is_upgrade_for(it):
159 it.upgrade = self.current_item
161 def create_item(self, name):
162 if name.endswith("Flask"):
165 def handle_value(self, column, data):
167 self.current_item.droplevel = int(data)
168 elif column==3 and not self.utility:
169 self.current_item.amount = int(data)
170 elif (column==4 and not self.utility) or (column==3 and self.utility):
171 self.current_item.duration = float(data)
173 class SkillGemListParser(WikiParser):
175 super(SkillGemListParser, self).__init__()
177 self.in_subheading = False
181 def handle_starttag(self, tag, attrs):
182 super(SkillGemListParser, self).handle_starttag(tag, attrs)
185 self.in_subheading = True
187 if self.in_list and not self.ignore:
192 def handle_endtag(self, tag):
193 super(SkillGemListParser, self).handle_endtag(tag)
196 self.in_subheading = False
198 def handle_data(self, data):
201 if self.in_subheading:
202 self.in_list = (data=="List")
204 class SkillGemParser(WikiParser):
205 prices = (("alchemy", "Orb of Alchemy"),
206 ("chance", "Orb of Chance"),
207 ("alteration", "Orb of Alteration"),
208 ("transmute", "Orb of Transmutation"),
209 ("wisdom", "Scroll of Wisdom"))
212 super(SkillGemParser, self).__init__()
215 self.in_heading = False
216 self.in_subheading = False
217 self.in_subheading = False
219 self.in_box_heading = False
220 self.in_purchase = False
221 self.in_progression = False
222 self.in_acquisition = False
223 self.have_acquisition_data = False
226 self.attribute_reqs = []
228 def handle_starttag(self, tag, attrs):
229 super(SkillGemParser, self).handle_starttag(tag, attrs)
232 self.in_heading = True
234 self.in_subheading = True
240 if n=="class" and "item-box" in v:
245 if n=="class" and "header" in v:
246 self.in_purchase = False
247 self.in_box_heading = True
253 elif tag=="td" or tag=="th":
256 if self.in_progression and self.column>=3:
259 self.attribute_reqs.append([v, 0])
262 if n=="id" and v=="footer":
263 if not self.have_acquisition_data:
264 self.item.drop_only = True
266 def handle_endtag(self, tag):
267 super(SkillGemParser, self).handle_endtag(tag)
270 self.in_heading = False
272 self.in_subheading = False
277 self.in_box_heading = False
279 if self.in_progression and self.row==2 and self.attribute_reqs:
280 self.attribute_reqs.sort(key=lambda r: r[1])
281 self.item.primary_attribute = self.attribute_reqs[0][0]
282 if len(self.attribute_reqs)>1:
283 self.item.secondary_attribute = self.attribute_reqs[1][0]
285 def handle_data(self, data):
292 paren = name.find('(')
294 name = name[:paren].strip()
295 self.item = SkillGem(name)
296 if name.startswith("Vaal"):
297 self.item.vaal = True
298 elif self.in_subheading:
299 self.in_progression = ("progression" in data)
300 self.in_acquisition = ("acquisition" in data)
301 elif self.in_box_heading:
302 if "Purchase" in data:
303 self.in_purchase = True
304 elif self.in_purchase:
306 for p, n in SkillGemParser.prices:
309 elif self.in_progression:
310 if self.row==2 and self.column>=3 and self.column<3+len(self.attribute_reqs):
311 self.attribute_reqs[self.column-3][1] = int(data)
312 elif self.in_acquisition:
313 self.have_acquisition_data = True
315 def scrape_flasks(out, url, kind):
316 r = requests.get(url)
318 p.feed(codecs.decode(r.content, r.encoding))
320 out.write('category "flask.{}.best_at_level"\n'.format(kind))
321 out.write('{\n\tclass "Flask";\n\tor\n\t{\n')
323 out.write("\t\tand\n\t\t{\n")
324 out.write('\t\t\tbase_type "{}";\n'.format(it.name))
326 out.write("\t\t\titem_level {} {};\n".format(it.droplevel, it.upgrade.droplevel-1))
328 out.write("\t\t\tmin_item_level {};\n".format(it.droplevel))
329 out.write("\t\t};\n")
330 out.write("\t};\n};\n")
333 r = requests.get("https://pathofexile.gamepedia.com/List_of_divination_cards")
334 p = DivinationCardsParser()
335 p.feed(codecs.decode(r.content, r.encoding))
340 by_reward.setdefault(it.reward_kind, []).append(it)
342 out = open("cards.txt", "w")
343 for r, il in by_reward.items():
344 out.write('category "card.{}"\n'.format(r))
345 out.write('{\n\tclass "Card";\n\tor\n\t{\n')
347 out.write('\t\tbase_type "{}";\n'.format(it.name))
348 out.write("\t};\n};\n")
350 out = open("flasks.txt", "w")
351 scrape_flasks(out, "https://pathofexile.gamepedia.com/Life_Flasks", "life")
352 scrape_flasks(out, "https://pathofexile.gamepedia.com/Mana_Flasks", "mana")
353 scrape_flasks(out, "https://pathofexile.gamepedia.com/Hybrid_Flasks", "hybrid")
355 p = FlasksParser(True)
356 r = requests.get("https://pathofexile.gamepedia.com/Utility_Flasks")
357 p.feed(codecs.decode(r.content, r.encoding))
358 r = requests.get("https://pathofexile.gamepedia.com/Critical_Utility_Flasks")
359 p.feed(codecs.decode(r.content, r.encoding))
361 out.write('category "flask.utility"\n{\n\tclass "Flask";\n\tor\n\t{\n')
363 out.write('\t\tbase_type "{}";\n'.format(it.name))
364 out.write("\t};\n};\n")
366 r = requests.get("https://pathofexile.gamepedia.com/List_of_skill_gems")
367 p = SkillGemListParser()
368 p.feed(codecs.decode(r.content, r.encoding))
374 r2 = requests.get("https://pathofexile.gamepedia.com"+l)
375 p2 = SkillGemParser()
376 p2.feed(codecs.decode(r2.content, r2.encoding))
379 sys.stdout.write(prefix+"{}\n".format(p2.item.name))
380 prefix = "\033[1A\033[K"
382 sys.stdout.write(prefix)
384 out = open("skillgems.txt", "w")
385 out.write('category "skillgem.special.vaal"\n{\n\tclass "Skill Gem";\n\tor\n\t{\n')
388 out.write('\t\tbase_type "{}";\n'.format(g.name))
389 out.write("\t};\n};\n")
391 out.write('category "skillgem.special.drop"\n{\n\tclass "Skill Gem";\n\tor\n\t{\n')
393 if (g.drop_only or not g.price) and not g.vaal:
394 out.write('\t\tbase_type "{}";\n'.format(g.name))
395 out.write("\t};\n};\n")
397 for p, n in SkillGemParser.prices:
398 out.write('category "skillgem.price.{}"\n'.format(p))
399 out.write('{\n\tclass "Skill Gem";\n\tor\n\t{\n')
401 if not g.drop_only and g.price==p:
402 out.write('\t\tbase_type "{}";\n'.format(g.name))
403 out.write("\t};\n};\n")
405 for a in ["strength", "dexterity", "intelligence"]:
406 out.write('category "skillgem.attribute.{}"\n'.format(a))
407 out.write('{\n\tclass "Skill Gem";\n\tor\n\t{\n')
409 if g.primary_attribute==a:
410 out.write('\t\tbase_type "{}";\n'.format(g.name))
411 out.write("\t};\n};\n")
413 if __name__=="__main__":