Tidak dapat menguraikan nama dari halaman ketiga dan seterusnya

Ini mulai berfungsi jika saya menghapus kunci muatan seperti dnn$ctr410$MemberSearch$grdMembers$ctl00$ctl02$ctl01$ctl02 yang merupakan kunci untuk tombol panah.

    name_length = len('dnn$ctr410$MemberSearch$grdMembers$ctl00$ctl02$ctl01$ctl02')

    for key in list(payload.keys()):
        if key.startswith('dnn') and len(key) == name_length:
            payload.pop(key)
            print(key)

tetapi Anda dapat menggunakan metode dari αԋɱҽԃ αмєяιcαη jawaban untuk memastikan Anda hanya mengirimkan nilai yang diperlukan.

import requests
from bs4 import BeautifulSoup

link = 'https://www.icsi.in/student/Members/MemberSearch.aspx?SkinSrc=%5BG%5DSkins/IcsiTheme/IcsiIn-Bare&ContainerSrc=%5BG%5DContainers/IcsiTheme/NoContainer'

with requests.Session() as s:
    s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
    r = s.get(link)
    soup = BeautifulSoup(r.text,"lxml")

    payload = {i['name']:i.get('value','') for i in soup.select('input[name]')}
    payload['__EVENTTARGET'] = 'dnn$ctr410$MemberSearch$btnSearch'

    page = 5
    while True:

        r = s.post(link, data=payload)
        soup = BeautifulSoup(r.text, "lxml")
        for item in soup.select("span[id$='_lblFullName']"):
            print(item.text)

        page += 2

        payload = {i['name']:i.get('value','') for i in soup.select('input[name]')}
        payload['__EVENTTARGET'] = 'dnn$ctr410$MemberSearch$grdMembers$ctl00$ctl02$ctl01$ctl{:02}'.format(page)

        name_length = len('dnn$ctr410$MemberSearch$grdMembers$ctl00$ctl02$ctl01$ctl02')
        for key in list(payload.keys()):
            if key.startswith('dnn') and len(key) == name_length:
                payload.pop(key)
                print(key)

        payload['__dnnVariable'] = {'__scdoff':'1','__dnn_pageload':'__dnn_setScrollTop();'}
        payload['ScrollTop'] = '400'

EDIT: halaman menggunakan sistem yang lebih kompleks dan setelah 10 halaman menampilkan link baru tetapi dengan nilai lama ctl07, ctl09. Alih-alih tautan ini saya menggunakan nama dari tombol dengan panah ke halaman berikutnya - pada awalnya memiliki nilai ctrl28 tetapi setelah 10 halaman memiliki ctrl30 (karena ada lebih banyak tautan - ia menambahkan tautan ... ke daftar 10 halaman berikutnya/sebelumnya)

import requests
from bs4 import BeautifulSoup

link = 'https://www.icsi.in/student/Members/MemberSearch.aspx?SkinSrc=%5BG%5DSkins/IcsiTheme/IcsiIn-Bare&ContainerSrc=%5BG%5DContainers/IcsiTheme/NoContainer'

with requests.Session() as s:
    s.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1; ) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
    r = s.get(link)
    soup = BeautifulSoup(r.text,"lxml")

    payload = {i['name']:i.get('value','') for i in soup.select('input[name]')}
    payload['__EVENTTARGET'] = 'dnn$ctr410$MemberSearch$btnSearch'

    page = 1  # I don't need it to generate lins, now I use it only to display page number
    while True:
        print('page:', page)
        page += 1

        r = s.post(link, data=payload)
        soup = BeautifulSoup(r.text, "lxml")
        for item in soup.select("span[id$='_lblFullName']"):
            print(item.text)

        payload = {i['name']:i.get('value','') for i in soup.select('input[name]')}

        name_length = len('dnn$ctr410$MemberSearch$grdMembers$ctl00$ctl02$ctl01$ctl28')
        for key in list(payload.keys()):
            if key.startswith('dnn') and len(key) == name_length:
                payload.pop(key)
                #print(key)

        # button with arrow to next page

        next_page = soup.select("input[class='rgPageNext']")
        if not next_page:
            break

        next_page = next_page[0]['name']
        print(next_page)
        payload[next_page] = ''

        payload['__dnnVariable'] = {'__scdoff':'1','__dnn_pageload':'__dnn_setScrollTop();'}
        payload['ScrollTop'] = '400'

Sebenarnya Anda perlu memasukkan parameter Payload postingan lengkap.

Kita harus melakukannya dalam Session yang sama menggunakan requests.Session() karena penomoran halaman situs web menggunakan rotasi function berdasarkan __dnnVariable yang mana server menerimanya berdasarkan permintaan JS yang diterjemahkan ke Loop.

Dimana sebenarnya arti Next

Jadi, saya telah membuat permintaan GET terlebih dahulu dan mengambil params yang diperlukan (beberapa di antaranya dinamis dan lainnya statis)

Kemudian, saya membuat permintaan posting di bawah session yang sama

import requests
import re
from bs4 import BeautifulSoup
from urllib.parse import unquote

data = {
    '__EVENTTARGET': "dnn$ctr410$MemberSearch$btnSearch",
    '__EVENTARGUMENT': '',
    '__VIEWSTATEENCRYPTED': '',
    'dnn$ctlHeader$dnnSearch$Search': 'SiteRadioButton',
    'dnn$ctlHeader$dnnSearch$txtSearch': '',
    'dnn$ctr410$MemberSearch$txtFirstName': '',
    'dnn$ctr410$MemberSearch$txtLastName': '',
    'dnn$ctr410$MemberSearch$ddlMemberType': 0,
    'dnn$ctr410$MemberSearch$txtMembershipNumber': '',
    'dnn$ctr410$MemberSearch$txtCpNumber': '',
    'dnn$ctr410$MemberSearch$txtCity': '',
    'dnn$ctr410$MemberSearch$txtOrganisation': '',
    'dnn$ctr410$MemberSearch$txtAddress2': '',
    'dnn$ctr410$MemberSearch$txtAddress3': '',
    'dnn$ctr410$MemberSearch$txtEmail': '',
    'dnn_ctr410_MemberSearch_grdMembers_ClientState': '',
    'ScrollTop': 432,
    '__dnnVariable': '{"__scdoff":"1","__dnn_pageload":"__dnn_setScrollTop();"}'
}


def main(url):
    with requests.Session() as req:
        r = req.get(url)
        soup = BeautifulSoup(r.content, 'html.parser')
        data['StylesheetManager_TSSM'] = re.search(
            r"hf.value \+= '(.*?)\'", r.text).group(1)
        data['ScriptManager_TSM'] = unquote(soup.findAll('script', src=True)
                                            [2]['src']).split("=", 3)[-1]
        data['__VIEWSTATE'] = soup.find("input", id="__VIEWSTATE").get("value")
        data['__VIEWSTATEGENERATOR'] = soup.find(
            "input", id="__VIEWSTATEGENERATOR").get("value")
        data['__EVENTVALIDATION'] = soup.find(
            "input", id="__EVENTVALIDATION").get("value")

        for _ in range(10):
            r = req.post(url, data=data)
            soup = BeautifulSoup(r.content, 'html.parser')
            names = [name.text for name in soup.select("div.name_head")]
            page = soup.select_one(
                "a.rgCurrentPage").next_sibling['href'].split("'")[1]
            data['__EVENTTARGET'] = page
            data['__EVENTVALIDATION'] = soup.find(
                "input", id="__EVENTVALIDATION").get("value")
            data['__VIEWSTATE'] = soup.find(
                "input", id="__VIEWSTATE").get("value")
            print(names)


main("https://www.icsi.in/student/Members/MemberSearch.aspx")

Catatan: pada setiap permintaan Anda akan mendapatkan data yang tidak berurutan karena fungsi search menggunakan random di server back-end.

Keluaran:

['SH. DILIP RAGHUNATH KOTWAL', 'SH. ARUNODAY ROY MUKHERJEE', 'SH. J SUBRAMANI', 'SH. R KRISHNAMANI', 'SH. R NARAYANASWAMI', 'SH. M V GOPALAKRISHNAN', 'SH. RAJAM KRISHNAMURTHY', 'SH. V SIVASUBRAMANIAN', 'SH. V RAGHAVENDRAN', 'SH. G V AIMAN']
['SH. K J MATHEW', 'SH. K K GHOSH', 'SH. SUBHASH CHANDER DHAWAN', 'SH. BABU RAM MAHESWARI', 'SH. S SWAMINATHAN', 'SH. T S A AIYER', 'SH. KOVILOOR VIJAYARAGHAVACHARI SAMPATHKUMAR', 'SH. M KRISHNAN', 'SH. R N BANSAL', 'SH. N V RAMAN']
['SH. R VENKATARAMANI', 'SH. UTPALENDU ROY CHOUDHURY', 'SH. LAKSHMI NARAYANAN V', 'SH. PARIJAT KUMAR HORE', 'SH. B R VENKATESAN', 'SH. KISHAN GOPAL SOMANI', 'SH. O P GANERIWALA', 'SH. P T KUPPUSWAMY', 'SH. U P MATHUR', 'SH. N N UPADHYAY']
['SH. N K BHANDARI', 'SH. S R C SETTY', 'SH. S V BALASUBRAMANIAN', 'SH. HOSHIE HIRJI MALGHAM', 'SH. KAIKOBAD SORABJI ITALIA', 'SH. K SIVADAS', 'SH. K K SIVARAMAKRISHNAN', 'SH. A CHANDRASEKARAN', 'SH. R PONNAMBALAM', 'SH. T K B VENKATARAMAN']
['SH. NARINDER PAL', 'SH. PARKASH ATAM', 'SH. K A PARTHASARATHY', 'SH. SURESH CHANDRA OSWAL', 'SH. MAHENDRA KANTILAL SHAH', 'SH. V. SANTHANAKRISHNA', 'SH. VASANT NARAYAN GOGATE', 'SH. MANEKLAL 
PATEL', 'SH. B N VISHWANATH', 'SH. B S L NARAYAN']
['SH. P L N VIJAYANAGAR', 'SH. SHREEPAD MARTAND  KORDE', 'SH. SHIV BHAGWAN KOTHARI', 'SH. R B POPLAI', 'SH. RAMESH KHANNA', 'SH. RAVINDER NATH JOSHI', 'SH. VIDYA SAGAR AGGARWAL', 'SH. ARVIND JAYKUMAR CHAKOTE', 'SH. V RAMASESHAN', 'SH. BADRINARAYAN BALDAWA']
['SH. C GOVINDANKUTTY', 'SH. A G MADHAVAN', 'SH. DHIRAJ NATH BHATTACHARYYA', 'SH. RAMESHWAR LAL INANI', 'SH. RAMESHWARDAS C DAGA', 'SH. R SUBRAMANIAN', 'SH. S M REGE', 'SH. NARENDRA KUMAR KAPOOR', 'SH. K RAMAMURTHI', 'SH. ROOPENDRA NARAYAN ROY']
['SH. KALYAN KUMAR MITRA', 'SH. KALYANASUNDARAM ', 'SH. N A SESHADRI', 'SH. RAJENDRA KUMAR JAIN', 'SH. BISWAJIT SEN', 'SH. RAMKRISHNA NATHOOMAL  AGRAWAL', 'SH. P C SHETH', 'SH. K S NATARAJAN', 
'SH. S N DAMLE', 'SH. A M FADIA']
['DR. K N M RAO', 'SH. IYER M. RAMASWAMY', 'SH. DILIP KANTI MAZUMDAR', 'SH. RAM CHANDRA NIGAM', 'SH. SUBRAHMANIAM VISWANATHAN', 'SH. SURESH KUMAR JERATH', 'SH. A Y SRINIVASAN', 'DR. S C GARG', 
'SH. CHANDRA PRAKASH SHARDA', 'SH. M P JAIN']
['SH. E S DWARKANATH', 'SH. MYSORE SHAMANNA  RAMACHANDRA', 'SH. SUBHASH CHANDER SINGHAL', 'SH. T T SINHA', 'SH. G R BHANDARI', 'SH. M P GOEL', 'SH. CHOKKANATHAPURAM SUBRAMANIAN  NATESAN', 'SH. 
V M PATEL', 'SH. BIJOY KUMAR AGARWALLA', 'SH. BAHADUR CHAND JAIN']