Skip to content

Update selectors #828

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
+246 −2
Open

Update selectors #828

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1775a6c
update selectors
addie9800 Nov 10, 2025
4c298f3
fix title
addie9800 Nov 10, 2025
77b7525
remove `removesuffix()`
addie9800 Nov 10, 2025
713c616
add `V1_1`
addie9800 Nov 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions src/fundus/publishers/ls/lesotho_times.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

class LesothoTimesParser(ParserProxy):
class V1(BaseParser):
VALID_UNTIL = datetime.date(2025, 9, 20)

_paragraph_selector = XPath("//div[@class='entry-content']/p[text() or span]")
_subheadline_selector = XPath(
"//div[@class='entry-content']/p[not(text() or em) and strong[not(em)] and position()>4]"
Expand All @@ -40,11 +42,15 @@ def publishing_date(self) -> Optional[datetime.datetime]:

@attribute
def authors(self) -> List[str]:
return generic_author_parsing(generic_nodes_to_text(self._author_selector(self.precomputed.doc)))
return generic_author_parsing(
generic_nodes_to_text(self._author_selector(self.precomputed.doc)), split_on=["/"]
)

@attribute
def title(self) -> Optional[str]:
return self.precomputed.meta.get("og:title")
if title := self.precomputed.meta.get("og:title"):
return title.replace("- Lesotho Times", "").strip()
return None

@attribute
def images(self) -> List[Image]:
Expand All @@ -54,3 +60,26 @@ def images(self) -> List[Image]:
image_selector=XPath("//div[@class='feature-postimg']/img"),
upper_boundary_selector=XPath("//header"),
)

class V1_1(V1):
VALID_UNTIL = datetime.date.today()

_paragraph_selector = XPath("//div[contains(@class,'entry-content')]/p[text() or span]")
_subheadline_selector = XPath(
"//div[contains(@class,'entry-content')]/p[not(text() or em) and strong[not(em)] and position()>4]"
)
_summary_selector = XPath("//div[contains(@class,'entry-content')]/p[not(text()) and (strong[em] or em)]")

_author_selector = XPath(
"//div[contains(@class,'entry-content')]/p[not(text() or em) and strong[not(em)] and position()<5]"
)

@attribute
def images(self) -> List[Image]:
return image_extraction(
doc=self.precomputed.doc,
paragraph_selector=self._paragraph_selector,
image_selector=XPath("//div[@class='feature-postimg' or contains(@class, 'post-image')]/img"),
caption_selector=XPath("./ancestor::div[contains(@class,'media')]//figcaption"),
upper_boundary_selector=XPath("//header"),
)
211 changes: 211 additions & 0 deletions tests/resources/parser/test_data/ls/LesothoTimes.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,5 +166,216 @@
],
"publishing_date": "2025-05-14 20:57:43+00:00",
"title": "From adversity to advocacy: a young lawyer's journey"
},
"V1_1": {
"authors": [
"Mathatisi Sebusi",
"Mohalenyane Phakela"
],
"body": {
"summary": [
"…as Bela Bela communities cry foul over lost livelihoods",
"…accuse chiefs of accepting bribes to ignore concerns"
],
"sections": [
{
"headline": [],
"paragraphs": [
"A Chinese-owned crusher and quarry mining company has come under heavy criticism from five Bela Bela villages who claim its blasting operations are destroying their homes and livelihoods.",
"Communities living near the site report that frequent powerful explosions from Moruo Mining and Construction Company have caused severe structural damages to their houses, which include cracked walls, weakened foundations and falling off plaster.",
"Mining debris is increasingly rendering fields unproductive and pastures unusable.",
"Residents of Ha Tuke, Mechaling, Ha Lephallo, Ha Pitso and Mokotane say they now live in fear, wondering for how much longer their homes will withstand the blasts.",
"Some have already been forced to relocate due to safety concerns yet so far the mining company has only resettled two households.",
"Despite the disruption and severe losses the host community has endured, the villagers claim Moruo has failed to make any meaningful developments or investments in the area.",
"Villagers argue that instead of benefitting from the mining activity, they are paying a heavy price through damaged property, noise pollution, unhealthy exposure to dust and a declining quality of life.",
"They insist that the company, which has been operating for four years and was recently awarded a fresh 10-year lease by the Ministry of Natural Resources, should be held accountable and be compelled to either repair the affected homes or adopt safer, community-friendly mining methods."
]
},
{
"headline": [
"“Kickbacks” for chiefs"
],
"paragraphs": [
"What frustrates the villagers the most, they say, is that their area chief, Seshophe Hlajoane, and Peete Lesaoana Peete, the Kueneng and Mapoteng Principal Chief, are allegedly taking bribes from the Chinese operator, Kang Shao Pin, who owns the mine.",
"They accuse the chiefs of sacrificing the interests of the very communities they are entrusted to protect for personal gain.",
"Residents say they were shocked to learn that Moruo’s mining licence had been extended for a further 10 years without any consultations while their chiefs are using community grievances to solicit money from Mr Pin.",
"The chiefs allegedly hold gatherings to gather villagers’ views but never act on them and, instead, nicodemously pass the complaints on to Mr Pin in exchange for favours.",
"Chief Hlajoane’s office, which is currently under construction, is reportedly being built by Mr Pin.",
"Meanwhile, Chief Peete is alleged to be receiving money earmarked for community development from Moruo but that money never reaches the intended beneficiaries.",
"Villagers also allege he receives a permanent three-percent quota for every load of crusher material leaving the mine.",
"During the two hours that the Lesotho Times team spent in the area, it witnessed five trucks laden with crusher material leaving the mining site."
]
},
{
"headline": [
"Pleas fall on deaf ears"
],
"paragraphs": [
"Former area councillor, Seabata Moeketsi, said they have repeatedly raised concerns with local leaders, including chiefs Peete and Hlajoane, but their pleas have fallen on deaf ears.",
"Mr Moeketsi, who previously worked for Moruo before falling out with the company for defending community interests, said the mining lease was initially issued in 2014 to one Mr Makama of Maseru, on condition that he would develop the area.",
"He said Mr Makama later subleased the operation to China Geo, which left four years ago after villagers began protesting and blocking trucks.",
"Mr Makama then sold the lease to Moruo, which was allowed to operate on the condition that it would electrify villages, provide water, repair roads and hire local labour.",
"“The four years have passed without any development. The only thing they did was bring a transformer, which currently only supplies electricity to two houses,” he said.",
"“Our houses are cracked and our fields are unploughable because they are filled with stones from the blasting. Apart from the noise, there is also dust from the mine which we believe will soon cause health hazards.”",
"Mr Moeketsi further accused the chiefs of being on Moruo’s payroll.",
"“These leaders know exactly what is going on. Every time we report our concerns, they pretend to listen and promise to act, but nothing ever happens. We suspect they deceive us only to go back to the company to take bribes and silence us.”",
"He added that some leaders now deny that any development agreements ever existed.",
"“I have heard that there was money given to Chief Peete for community development; I was told it was deposited directly into his personal account and has never been used to uplift the villages.”",
"He also reiterated claims that Chief Peete receives three percent of the revenue from every truckload of crusher stone extracted from the mine.",
"Mr Moeketsi claimed that whenever residents attempt to protest or demand compliance from the company, police officers and soldiers are deployed to intimidate them.",
"Even more concerning, he said, is that the mining lease — which expired in August — was renewed without consultation.",
"“How does a mining lease get renewed for a company that failed to meet its obligations in the first place? They didn’t even talk to us — we just saw them continue operations.”",
"He added that: “Mr Pin always tells us to go anywhere we think we will get help, and claims that ministers are his friends.”",
"These sentiments were echoed by several community members interviewed during the Lesotho Times visit."
]
},
{
"headline": [
"Water woes"
],
"paragraphs": [
"World Vision Lesotho previously brought hope to the villages by erecting a borehole to provide clean water. But villagers say Moruo has since hijacked the water supply as the company draws water from the same borehole, forcing them to drink from unprotected sources since the it quickly runs out.",
"“Instead of constructing a water system, they use our only water source, overwhelming it and leaving us without water for days,” Mr Moeketsi said."
]
},
{
"headline": [
"Granny (78) suffers in silence"
],
"paragraphs": [
"Seventy-eight-year-old ’Manteboheng Molatjeng sits outside her cracked brick house, located just 200 metres from the blasting site. Trucks constantly pass along the gravel road besides her yard, each time raising clouds of dust that engulf her home.",
"“This is our daily life now. We breathe dust every day. These stones you see are from the mine. My windows haven’t broken yet, but it’s just a matter of time. The cracks in my house widen with every blast,” Ms Molatjeng said.",
"She has already been forced to sell her field.",
"“It is heartbreaking. They put prices on our land without considering that we have been surviving on these fields for years. I had a big field which the Chinese forced me to sell at a mere M50 000. Without these fields, our livelihoods are gone. I now only survive on my pension, which is too little.”",
"At her advanced age, she said, the water shortages are unbearable.",
"“If I don’t collect enough water before the mine exhausts it, I must go to the well down the valley. Sometimes I go for days without water. Imagine someone my age carrying such heavy loads.”",
"Another villager, Lipolelo Potobere, said they often go for days without water and are forced to rely on unsafe sources.",
"“As we speak, we have been without water for three days. We are collecting water from an unprotected source that exposes us to diseases. When we protest, police and soldiers are called to stop us.”",
"She added that the promised jobs never materialised.",
"“I think only two people from here work at the mine.”"
]
},
{
"headline": [
"Chiefs respond"
],
"paragraphs": [
"Chief Hlajoane said he was aware of the community’s suffering but denied ever taking bribes.",
"“I like money and will not refuse it if the Chinese offer it. But what I do not do is take or demand money for my community to be silenced,” Chief Hlajoane said.",
"He admitted that the company is building him an office but insisted it was out of generosity, not corruption.",
"“He is going to build similar offices in Ha Lephallo, Ha Pitso and Mokotane as part of CSI (Corporate Social Investment), so there is nothing sinister there.”",
"He also acknowledged that the licence was renewed without consultation and that the company has not fulfilled its development obligations.",
"“They brought a transformer so families only need to extend the supply to their houses. They are unhappy because they want the company to install electricity in every household, which is unreasonable,” he said.",
"He added that he reported the community’s grievances to the Berea District Administrator (DA) and is awaiting a response.",
"Principal Chief Lesaoana promised to respond but did not do so. Subsequent attempts to reach him were unsuccessful as his mobile phone rang unanswered.",
"Berea DA Phahlane Mokoko confirmed receiving a letter from Chief Hlajoane but said he could not accept it because it did not follow protocol."
]
},
{
"headline": [
"Operator denies allegations"
],
"paragraphs": [
"At the mine site, trucks were being loaded amid thick clouds of dust. Security personnel said Mr Pin was unavailable.",
"When contacted, Mr Pin denied all the allegations and reacted angrily when asked about development commitments.",
"“I don’t know what you are talking about. These are all lies. And that water source they complain about — I am the one who repaired it after it was vandalised by thieves,” he said.",
"China Geo, for its part, told the Lesotho Times that it does not owe the community any development, saying obligations were tied to the mining lease owner, whose identity it could not disclose."
]
},
{
"headline": [
"The government’s stance"
],
"paragraphs": [
"Minister of Natural Resources, Mohlomi Moleko, would not confirm whether the ministry is aware of the situation and only said: “Ask them to submit a formal complaint to the ministry”."
]
}
]
},
"images": [
{
"versions": [
{
"url": "https://lestimes.com/wp-content/uploads/2025/11/20251117_133533-300x172.png",
"query_width": null,
"size": {
"width": 300,
"height": 172
},
"type": "image/png"
},
{
"url": "https://lestimes.com/wp-content/uploads/2025/11/20251117_133533-585x334.png",
"query_width": null,
"size": {
"width": 585,
"height": 335
},
"type": "image/png"
},
{
"url": "https://lestimes.com/wp-content/uploads/2025/11/20251117_133533-588x336.png",
"query_width": null,
"size": {
"width": 588,
"height": 336
},
"type": "image/png"
},
{
"url": "https://lestimes.com/wp-content/uploads/2025/11/20251117_133533-700x400.png",
"query_width": null,
"size": {
"width": 700,
"height": 400
},
"type": "image/png"
},
{
"url": "https://lestimes.com/wp-content/uploads/2025/11/20251117_133533-768x439.png",
"query_width": null,
"size": {
"width": 768,
"height": 439
},
"type": "image/png"
},
{
"url": "https://lestimes.com/wp-content/uploads/2025/11/20251117_133533-960x549.png",
"query_width": null,
"size": {
"width": 960,
"height": 549
},
"type": "image/png"
},
{
"url": "https://lestimes.com/wp-content/uploads/2025/11/20251117_133533-1536x878.png",
"query_width": null,
"size": {
"width": 1536,
"height": 878
},
"type": "image/png"
},
{
"url": "https://lestimes.com/wp-content/uploads/2025/11/20251117_133533.png",
"query_width": null,
"size": {
"width": 1800,
"height": 1029
},
"type": "image/png"
}
],
"is_cover": true,
"description": null,
"caption": null,
"authors": [],
"position": 262
}
],
"publishing_date": "2025-11-21 13:34:26+00:00",
"title": "Mining blasts shatter homes"
}
}
Binary file added tests/resources/parser/test_data/ls/LesothoTimes_2025_11_24.html.gz
View file
Open in desktop
Binary file not shown.
4 changes: 4 additions & 0 deletions tests/resources/parser/test_data/ls/meta.info
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,9 @@
"LesothoTimes_2025_05_23.html.gz": {
"url": "https://lestimes.com/from-adversity-to-advocacy-a-young-lawyers-journey/",
"crawl_date": "2025-05-23 00:04:42.557906"
},
"LesothoTimes_2025_11_24.html.gz": {
"url": "https://lestimes.com/mining-blasts-shatter-homes/",
"crawl_date": "2025-11-24 15:25:14.010289"
}
}