diff options
Diffstat (limited to 'deluge/metafile.py')
| -rw-r--r-- | deluge/metafile.py | 460 |
1 files changed, 460 insertions, 0 deletions
diff --git a/deluge/metafile.py b/deluge/metafile.py new file mode 100644 index 0000000..81a371f --- /dev/null +++ b/deluge/metafile.py @@ -0,0 +1,460 @@ +# +# Original file from BitTorrent-5.3-GPL.tar.gz +# Copyright (C) Bram Cohen +# +# Modifications for use in Deluge: +# Copyright (C) 2007 Andrew Resch <andrewresch@gmail.com> +# +# This file is part of Deluge and is licensed under GNU General Public License 3.0, or later, with +# the additional special exception to link portions of this program with the OpenSSL library. +# See LICENSE for more details. +# + +import copy +import logging +import os.path +import time +from enum import Enum +from hashlib import sha1 as sha +from hashlib import sha256 + +import deluge.component as component +from deluge.bencode import bencode +from deluge.common import utf8_encode_structure +from deluge.event import CreateTorrentProgressEvent + +log = logging.getLogger(__name__) + +ignore = ['core', 'CVS', 'Thumbs.db', 'desktop.ini'] + +noncharacter_translate = {} +for i in range(0xD800, 0xE000): + noncharacter_translate[i] = ord('-') +for i in range(0xFDD0, 0xFDF0): + noncharacter_translate[i] = ord('-') +for i in (0xFFFE, 0xFFFF): + noncharacter_translate[i] = ord('-') + + +def gmtime(): + return time.mktime(time.gmtime()) + + +def dummy(*v): + pass + + +class TorrentFormat(str, Enum): + V1 = 'v1' + V2 = 'v2' + HYBRID = 'hybrid' + + @classmethod + def _missing_(cls, value): + if not value: + return None + + value = value.lower() + for member in cls: + if member.value == value: + return member + + def to_lt_flag(self): + if self.value == 'v1': + return 64 + if self.value == 'v2': + return 32 + return 0 + + def includes_v1(self): + return self == self.__class__.V1 or self == self.__class__.HYBRID + + def includes_v2(self): + return self == self.__class__.V2 or self == self.__class__.HYBRID + + +class RemoteFileProgress: + def __init__(self, session_id): + self.session_id = session_id + + def __call__(self, piece_count, num_pieces): + component.get('RPCServer').emit_event_for_session_id( + self.session_id, CreateTorrentProgressEvent(piece_count, num_pieces) + ) + + +def make_meta_file_content( + path, + url, + piece_length, + progress=None, + title=None, + comment=None, + safe=None, + content_type=None, + webseeds=None, + name=None, + private=False, + created_by=None, + trackers=None, + torrent_format=TorrentFormat.V1, +): + data = {'creation date': int(gmtime())} + if url: + data['announce'] = url.strip() + + if progress is None: + progress = dummy + try: + session_id = component.get('RPCServer').get_session_id() + except KeyError: + pass + else: + if session_id: + progress = RemoteFileProgress(session_id) + + info, piece_layers = makeinfo( + path, + piece_length, + progress, + name, + content_type, + private, + torrent_format, + ) + + # check_info(info) + data['info'] = info + if piece_layers is not None: + data['piece layers'] = piece_layers + if title: + data['title'] = title.encode('utf8') + if comment: + data['comment'] = comment.encode('utf8') + if safe: + data['safe'] = safe.encode('utf8') + + httpseeds = [] + url_list = [] + + if webseeds: + for webseed in webseeds: + if webseed.endswith('.php'): + httpseeds.append(webseed) + else: + url_list.append(webseed) + + if url_list: + data['url-list'] = url_list + if httpseeds: + data['httpseeds'] = httpseeds + if created_by: + data['created by'] = created_by.encode('utf8') + + if trackers and (len(trackers[0]) > 1 or len(trackers) > 1): + data['announce-list'] = trackers + + data['encoding'] = 'UTF-8' + return bencode(utf8_encode_structure(data)) + + +def default_meta_file_path(content_path): + a, b = os.path.split(content_path) + if b == '': + f = a + '.torrent' + else: + f = os.path.join(a, b + '.torrent') + return f + + +def make_meta_file( + path, + url, + piece_length, + progress=None, + title=None, + comment=None, + safe=None, + content_type=None, + target=None, + webseeds=None, + name=None, + private=False, + created_by=None, + trackers=None, +): + if not target: + target = default_meta_file_path(path) + + file_content = make_meta_file_content( + path, + url, + piece_length, + progress=progress, + title=title, + comment=comment, + safe=safe, + content_type=content_type, + webseeds=webseeds, + name=name, + private=private, + created_by=created_by, + trackers=trackers, + ) + + with open(target, 'wb') as file_: + file_.write(file_content) + + +def calcsize(path): + total = 0 + for s in subfiles(os.path.abspath(path)): + total += os.path.getsize(s[1]) + return total + + +def _next_pow2(num): + import math + + if not num: + return 1 + return 2 ** math.ceil(math.log2(num)) + + +def _sha256_merkle_root(leafs, nb_leafs, padding, in_place=True) -> bytes: + """ + Build the root of the merkle hash tree from the (possibly incomplete) leafs layer. + If len(leafs) < nb_leafs, it will be padded with the padding repeated as many times + as needed to have nb_leafs in total. + """ + if not in_place: + leafs = copy.copy(leafs) + + while nb_leafs > 1: + nb_leafs = nb_leafs // 2 + for i in range(nb_leafs): + node1 = leafs[2 * i] if 2 * i < len(leafs) else padding + node2 = leafs[2 * i + 1] if 2 * i + 1 < len(leafs) else padding + h = sha256(node1) + h.update(node2) + if i < len(leafs): + leafs[i] = h.digest() + else: + leafs.append(h.digest()) + return leafs[0] if leafs else padding + + +def _sha256_buffer_blocks(buffer, block_len): + import math + + nb_blocks = math.ceil(len(buffer) / block_len) + blocks = [ + sha256(buffer[i * block_len : (i + 1) * block_len]).digest() + for i in range(nb_blocks) + ] + return blocks + + +def makeinfo_lt( + path, piece_length, name=None, private=False, torrent_format=TorrentFormat.V1 +): + """ + Make info using via the libtorrent library. + """ + from deluge._libtorrent import lt + + if not name: + name = os.path.split(path)[1] + + fs = lt.file_storage() + if os.path.isfile(path): + lt.add_files(fs, path) + else: + for p, f in subfiles(path): + fs.add_file(os.path.join(name, *p), os.path.getsize(f)) + torrent = lt.create_torrent( + fs, piece_size=piece_length, flags=torrent_format.to_lt_flag() + ) + + lt.set_piece_hashes(torrent, os.path.dirname(path)) + torrent.set_priv(private) + + t = torrent.generate() + info = t[b'info'] + pieces_layers = t.get(b'piece layers', None) + + return info, pieces_layers + + +def makeinfo( + path, + piece_length, + progress, + name=None, + content_type=None, + private=False, + torrent_format=TorrentFormat.V1, +): + # HEREDAVE. If path is directory, how do we assign content type? + + v2_block_len = 2**14 # 16 KiB + v2_blocks_per_piece = 1 + v2_block_padding = b'' + v2_piece_padding = b'' + if torrent_format.includes_v2(): + if _next_pow2(piece_length) != piece_length or piece_length < v2_block_len: + raise ValueError( + 'Bittorrent v2 piece size must be a power of 2; and bigger than 16 KiB' + ) + + v2_blocks_per_piece = piece_length // v2_block_len + v2_block_padding = bytes(32) # 32 = size of sha256 in bytes + v2_piece_padding = _sha256_merkle_root( + [], nb_leafs=v2_blocks_per_piece, padding=v2_block_padding + ) + + path = os.path.abspath(path) + files = [] + pieces = [] + file_tree = {} + piece_layers = {} + if os.path.isdir(path): + if not name: + name = os.path.split(path)[1] + subs = subfiles(path) + if torrent_format.includes_v2(): + subs = sorted(subs) + length = None + totalsize = 0.0 + for p, f in subs: + totalsize += os.path.getsize(f) + else: + name = os.path.split(path)[1] + subs = [([name], path)] + length = os.path.getsize(path) + totalsize = length + is_multi_file = len(subs) > 1 + sh = sha() + done = 0 + totalhashed = 0 + + next_progress_event = piece_length + for p, f in subs: + file_pieces_v2 = [] + pos = 0 + size = os.path.getsize(f) + p2 = [n.encode('utf8') for n in p] + if content_type: + files.append( + {b'length': size, b'path': p2, b'content_type': content_type} + ) # HEREDAVE. bad for batch! + else: + files.append({b'length': size, b'path': p2}) + with open(f, 'rb') as file_: + while pos < size: + to_read = min(size - pos, piece_length) + buffer = memoryview(file_.read(to_read)) + pos += to_read + + if torrent_format.includes_v1(): + a = piece_length - done + for sub_buffer in (buffer[:a], buffer[a:]): + if sub_buffer: + sh.update(sub_buffer) + done += len(sub_buffer) + + if done == piece_length: + pieces.append(sh.digest()) + done = 0 + sh = sha() + if torrent_format.includes_v2(): + block_hashes = _sha256_buffer_blocks(buffer, v2_block_len) + num_leafs = v2_blocks_per_piece + if size <= piece_length: + # The special case when the file is smaller than a piece: only pad till the next power of 2 + num_leafs = _next_pow2(len(block_hashes)) + root = _sha256_merkle_root( + block_hashes, num_leafs, v2_block_padding, in_place=True + ) + file_pieces_v2.append(root) + + totalhashed += to_read + if totalhashed >= next_progress_event: + next_progress_event = totalhashed + piece_length + progress(totalhashed, totalsize) + + if torrent_format == TorrentFormat.HYBRID and is_multi_file and done > 0: + # Add padding file to force piece-alignment + padding = piece_length - done + sh.update(bytes(padding)) + files.append( + { + b'length': padding, + b'attr': b'p', + b'path': [b'.pad', str(padding).encode()], + } + ) + pieces.append(sh.digest()) + done = 0 + sh = sha() + + if torrent_format.includes_v2(): + # add file to the `file tree` and, if needed, to the `piece layers` structures + pieces_root = _sha256_merkle_root( + file_pieces_v2, + _next_pow2(len(file_pieces_v2)), + v2_piece_padding, + in_place=False, + ) + dst_directory = file_tree + for directory in p2[:-1]: + dst_directory = dst_directory.setdefault(directory, {}) + dst_directory[p2[-1]] = { + b'': { + b'length': size, + b'pieces root': pieces_root, + } + } + if len(file_pieces_v2) > 1: + piece_layers[pieces_root] = b''.join(file_pieces_v2) + + if done > 0: + pieces.append(sh.digest()) + progress(totalsize, totalsize) + + info = { + b'piece length': piece_length, + b'name': name.encode('utf8'), + } + if private: + info[b'private'] = 1 + if content_type: + info[b'content_type'] = content_type + if torrent_format.includes_v1(): + info[b'pieces'] = b''.join(pieces) + if is_multi_file: + info[b'files'] = files + else: + info[b'length'] = length + if torrent_format.includes_v2(): + info.update( + { + b'meta version': 2, + b'file tree': file_tree, + } + ) + return info, piece_layers if torrent_format.includes_v2() else None + + +def subfiles(d): + r = [] + stack = [([], d)] + while stack: + p, n = stack.pop() + if os.path.isdir(n): + for s in os.listdir(n): + if s not in ignore and not s.startswith('.'): + stack.append((p + [s], os.path.join(n, s))) + else: + r.append((p, n)) + return r |