How do I create a dataframe of jobs and companies that includes hyperlinks?

I hope this helps . I would regex out from html returned the required info and construct the url from the parameters the page javascript uses to dynamically construct each url. Interestingly, the total number of listings is different when using requests than using browser. You can manually enter the number of listings e.g. 6175 (currently) or use the number returned by the request (which is lower and you miss some results). You could also use selenium to get the correct initial result count). You can then issue requests with offsets to get all listings.
Listings can be randomized in terms of ordering.

https://www.indeed.com/jobs?q=software+developer&l=San+Francisco&limit=50&start=0

import requests, re, hjson, math
import pandas as pd
from bs4 import BeautifulSoup as bs

p = re.compile(r"jobmap\[\d+\]= ({.*?})")
p1 = re.compile(r"var searchUID = '(.*?)';") 
counter = 0 
final = {}

with requests.Session() as s:
    r = s.get('https://www.indeed.com/q-software-developer-l-San-Francisco-jobs.html#')
    soup = bs(r.content, 'lxml')
    tk = p1.findall(r.text)[0] 
    listings_per_page = 10
    number_of_listings = int(soup.select_one('[name=description]')['content'].split(' ')[0].replace(',',''))
    #number_of_pages = math.ceil(number_of_listings/listings_per_page)
    number_of_pages =  math.ceil(6175/listings_per_page) #manually calculated
    for page in range(1, number_of_pages + 1):
        if page > 1:
            r = s.get('https://www.indeed.com/jobs?q=software+developer&l=San+Francisco&start={}'.format(10*page-1))
            soup = bs(r.content, 'lxml')
            tk = p1.findall(r.text)[0] 

        for item in p.findall(r.text):
            data = hjson.loads(item)
            jk = data['jk']
            row = {'title' : data['title']
               ,'company' : data['cmp']
               ,'url' : f'https://www.indeed.com/viewjob?jk={jk}&tk={tk}&from=serp&vjs=3'
              }
            final[counter] = row
            counter+=1

df = pd.DataFrame(final)
output_df = df.T
output_df.to_csv(r'C:\Users\User\Desktop\Data.csv', sep=',', encoding='utf-8-sig',index = False )

import requests, re, hjson, math
import pandas as pd
from bs4 import BeautifulSoup as bs
from selenium import webdriver
from selenium.webdriver.chrome.options import Options

options = Options()  
options.add_argument("--headless") 
d = webdriver.Chrome(r'C:\Users\HarrisQ\Documents\chromedriver.exe', options = options)
d.get('https://www.indeed.com/q-software-developer-l-San-Francisco-jobs.html#')
number_of_listings = int(d.find_element_by_css_selector('[name=description]').get_attribute('content').split(' ')[0].replace(',',''))
d.quit()
p = re.compile(r"jobmap\[\d+\]= ({.*?})")
p1 = re.compile(r"var searchUID = '(.*?)';") 
counter = 0 
final = {}

with requests.Session() as s:
    r = s.get('https://www.indeed.com/q-software-developer-l-San-Francisco-jobs.html#')
    soup = bs(r.content, 'lxml')
    tk = p1.findall(r.text)[0] 
    listings_per_page = 10
    number_of_pages =  math.ceil(6175/listings_per_page) #manually calculated
    for page in range(1, number_of_pages + 1):
        if page > 1:
            r = s.get('https://www.indeed.com/jobs?q=software+developer&l=San+Francisco&start={}'.format(10*page-1))
            soup = bs(r.content, 'lxml')
            tk = p1.findall(r.text)[0] 

        for item in p.findall(r.text):
            data = hjson.loads(item)
            jk = data['jk']
            row = {'title' : data['title']
               ,'company' : data['cmp']
               ,'url' : f'https://www.indeed.com/viewjob?jk={jk}&tk={tk}&from=serp&vjs=3'
              }
            final[counter] = row
            counter+=1

df = pd.DataFrame(final)
output_df = df.T
output_df.to_csv(r'C:\Users\User\Desktop\Data.csv', sep=',', encoding='utf-8-sig',index = False )