Extract links using link extractor




import scrapy

from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule


class AntariSpider(CrawlSpider):
    name = "antari"
    start_urls = ['https://antari.com/all-products/']

    rules = (
        Rule(
            LinkExtractor(
                restrict_css='.elementor-post__title a'
            ),
            callback="parse_item",
        ),
    )

    def parse_item(self, response):
        # Extract the link and text from the response
        link = response.url

        yield {
            'link': link,
        }

No comments

Powered by Blogger.