全桟知识体系（三）

(ts, aid)
需要2个参数
ts：当前时间戳
aid：分类id

if not is_existed('albums/%d.json' % cid):
    print(cid, '------------------start')
    while True:
        ts = 'ts-' + str(int(round(time.time() * 1000)))
        url = '/mobile/discovery/v2/category/metadata/albums/%s?calcDimension=hot' \
              '&categoryId=%d&device=iPhone&pageId=%d&pageSize=20&version=6.5.30' \
              % (ts, cid, page)
        data = get_url(url)
        albums_list = data.get('list', [])
        if albums_list:
            page += 1
            albums = albums + albums_list
        else:
            print(cid, '------------------ending')
            break
    print(cid, '----------------专栏数:', len(albums))
    save_json('albums/', '%d.json' % cid, {'albums': albums})

import requests, random, json, os


# 通用的获取接口
def get_url(url):
    headers = {
        'host': 'mobile.ximalaya.com',
        'accept': '*/*',
        'user-agent': 'ting_v6.5.30_c5(CFNetwork, iOS 12.1, iPhone9,1)',
        'accept-language': 'zh-cn',
        'accept-encoding': 'gzip, deflate',
        'connection': 'keep-alive'
    }
    host_ips = [
        'http://180.153.255.6',
        'http://114.80.170.74',
        'http://114.80.161.20',
        'http://114.80.161.18',
        'http://114.80.142.163'
    ]
    index = random.randint(0, 4)
    new_ip = host_ips[index]
    if 'mobile.ximalaya.com' in url:
        url = url
    else:
        url = new_ip + url
    try:
        print(url)
        response = requests.get(url=url, headers=headers)
        return response.json()
    except Exception as e:
        print('request Exception')
        print('sleep--------------------------------------10 seconds')
        time.sleep(10)
        os.system('python3 /Users/wuwei/python/xmly_spider/review.py')


# 通用的写入json
def save_json(folder, file_name, json_data):
    folder = '../xmly/' + folder
    file = os.path.join(folder, file_name)
    if not os.path.exists(folder):
        os.makedirs(folder)
    if not os.path.exists(file):
        with open(file, 'wb') as f:
            f.write(json.dumps(json_data).encode('utf-8'))


# 通用的读取json
def read_json(folder, file_name):
    folder = '../xmly/' + folder
    file = os.path.join(folder, file_name)
    if os.path.exists(file):
        with open(file, 'rb') as f:
            line_bytes = f.read()
            data = json.loads(line_bytes.decode('utf-8'))
            return data['data']


# 通用的是否存在
def is_existed(file):
    return os.path.exists('../xmly/'+ file)

# 通用的json数组去重
def distinct(json_arr):
    return [dict(t) for t in set([tuple(d.items()) for d in json_arr])]


# 通用的json转换
def to_album(album, album_extra, user, category_id):
    # 无标签，特殊处理，默认小说
    if 'showTagList' not in album_extra.keys():
        album_extra['showTagList'] = [{
            'tagId': 1,
            'tagName': '小说'
        }]
    # 无图片，特殊处理，默认为空字符串
    cover = album.get('coverSmall', '')
    if cover:
        if 'imagev2.xmcdn.com' in cover:
            cover = cover[cover.index('group'):cover.rindex('!')]
    data_album = {
        '_id': album['albumId'],
        'title': album['title'],
        'subTitle': album.get('intro', ''),
        'playsCounts': album['playsCounts'],
        'cover': cover,
        'categoryId': category_id,
        'uid': user['uid'],
        'showTagList': to_tag_id(album_extra['showTagList']),
        'intro': album_extra['intro']
    }
    return data_album


def to_user(user):
    avator = user.get('smallLogo', '')
    if avator:
        if 'imagev2.xmcdn.com' in avator:
            avator = avator[avator.index('group'):avator.rindex('!')]
    user_new = {
        '_id': user['uid'],
        'nickname': user['nickname'],
        'avator': avator,

    }
    return user_new


def to_tag(tags):
    tag_new = []
    for tag in tags:
        tag_new.append({'_id': tag['tagId'], 'name': tag['tagName']})
    return tag_new


def to_tag_id(tags):
    tag_new = []
    for tag in tags:
        tag_new.append(tag['tagId'])
    return tag_new

# 获取所有专栏
def read_all_albums():
    albums = read_json('detail/', 'album.json')
    for album in albums:
        aid = album['_id']
        if not is_existed('track_ids/%d.json' % aid):
            album_tracks = get_tracks(aid)
            save_json('track_ids/', '%d.json' % aid, album_tracks[0])
            save_json('tracks/', '%d.json' % aid, album_tracks[1])
        else:
            print('album_id:%d------------------album is_existed' % aid)
            

# 获取该专栏的所有音频
def get_tracks(aid):
    page = 1
    tracks = []
    tracks_id = []
    print(aid, '------------------start')
    while True:
        ts = 'ts-' + str(int(round(time.time() * 1000)))
        url = '/mobile/v1/album/track/%s?albumId=%d&device=iPhone&isAsc=true&isQueryInvitationBrand=true&' \
              'pageId=%d&pageSize=20' % (ts, aid, page)
        data = get_url(url)
        if data['ret'] == 0:
            tracks_list = data['data'].get('list', [])
            if tracks_list:
                page += 1
                for track in tracks_list:
                    tracks_id.append(track['trackId'])
                    play_free = track.get('playPathAacv224', track.get('playUrl64', ''))
                    # 没有免费的音频，通过VIP爬取
                    if not play_free:
                        track_url = get_track_url(aid, track['trackId'])
                        track['playPathAacv224'] = track_url
                    tracks.append(to_track(track))
            else:
                print(aid, '------------------ending')
                break
        else:
            print(aid, '------------------ending已下架')
            break
    print(aid, '----------------音频数:', len(tracks))
    return [tracks_id, tracks]
    
# 获取单个音频链接（VIP）
def get_track_url(aid, tid):
    ts = 'ts-' + str(int(round(time.time() * 1000)))
    url = '/mobile/download/v1/%d/track/%d/%s?trackQualityLevel=0' % (aid, tid, ts)
    data = get_url(url)
    if data['ret'] == 0:
        return data.get('downloadAacUrl', data.get('downloadUrl', ''))
    else:
        return ''