From 1f859997f4b1d3603c8941c29a4945ea9d5f7ec6 Mon Sep 17 00:00:00 2001 From: Vologhat <47892605+Vologhat@users.noreply.github.com> Date: Sun, 10 Jul 2022 14:11:27 +0300 Subject: [PATCH] M3U8 Dumper --- .../lite/downloaders/music/KeyType.java | 6 - .../vtosters/lite/downloaders/music/M3U8.java | 179 ------------------ .../lite/downloaders/music/M3UDownloader.java | 55 ++++++ .../lite/downloaders/music/Property.java | 140 -------------- .../lite/downloaders/music/TSDownload.java | 112 ----------- .../downloaders/music/utils/ByteUtils.java | 18 -- .../vtosters/lite/music/TransportStream.java | 36 ++++ .../ru/vtosters/lite/music/VKM3UParser.java | 65 +++++++ .../ru/vtosters/lite/utils/StreamUtils.java | 65 +++++++ 9 files changed, 221 insertions(+), 455 deletions(-) delete mode 100644 app/src/main/java/ru/vtosters/lite/downloaders/music/KeyType.java delete mode 100644 app/src/main/java/ru/vtosters/lite/downloaders/music/M3U8.java create mode 100644 app/src/main/java/ru/vtosters/lite/downloaders/music/M3UDownloader.java delete mode 100644 app/src/main/java/ru/vtosters/lite/downloaders/music/Property.java delete mode 100644 app/src/main/java/ru/vtosters/lite/downloaders/music/TSDownload.java delete mode 100644 app/src/main/java/ru/vtosters/lite/downloaders/music/utils/ByteUtils.java create mode 100644 app/src/main/java/ru/vtosters/lite/music/TransportStream.java create mode 100644 app/src/main/java/ru/vtosters/lite/music/VKM3UParser.java create mode 100644 app/src/main/java/ru/vtosters/lite/utils/StreamUtils.java diff --git a/app/src/main/java/ru/vtosters/lite/downloaders/music/KeyType.java b/app/src/main/java/ru/vtosters/lite/downloaders/music/KeyType.java deleted file mode 100644 index 7a8976a50..000000000 --- a/app/src/main/java/ru/vtosters/lite/downloaders/music/KeyType.java +++ /dev/null @@ -1,6 +0,0 @@ -package ru.vtosters.lite.downloaders.music; - -public enum KeyType { - NONE, - AES128 -} diff --git a/app/src/main/java/ru/vtosters/lite/downloaders/music/M3U8.java b/app/src/main/java/ru/vtosters/lite/downloaders/music/M3U8.java deleted file mode 100644 index e6b700b63..000000000 --- a/app/src/main/java/ru/vtosters/lite/downloaders/music/M3U8.java +++ /dev/null @@ -1,179 +0,0 @@ -package ru.vtosters.lite.downloaders.music; - -import android.util.Log; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Map; - -import ru.vtosters.lite.net.NetClient; -import ru.vtosters.lite.net.NetRequest; - -public class M3U8 { - private final String url; - private final OutputStream output; - private final Callback callback; - private final int returnChunk; - - public M3U8(String url, String savedFileName, OutputStream output, Callback callback, int chunksNum) { - try { - this.url = url; - var file = new File(savedFileName).getAbsoluteFile(); - file.getParentFile().mkdirs(); - this.output = new FileOutputStream(file); - this.callback = callback; - this.returnChunk = chunksNum - 1; - } catch (FileNotFoundException e) { - throw new RuntimeException(e); - } - } - - public static InputStream getStream(NetClient client, URL url) { - var request = new NetRequest.Builder() - .url(url.toString()) - .build(); - - try { - var response = client.newCall(request).execute(); - - if (!response.isSuccessful()) { - return null; - } - return response.getDataStream(); - } catch (IOException e) { - return null; - } - } - - // пожалуйста не смотрите сюда - public boolean run(NetClient client) { - try { - URL mediaURL; - URL m3u8Url = new URL(url); - int headerSize = 0; - try (InputStream is = getStream(client, m3u8Url)) { - if (is == null) { - return false; - } - try (BufferedReader br = new BufferedReader(new InputStreamReader(is))) { - String line; - ArrayList> urls = new ArrayList<>(); - long newUrl = 0; - while ((line = br.readLine()) != null) { - headerSize += line.getBytes().length; - line = line.trim(); - Property property = Property.checkProperty(line); - if (property == null) { - newUrl = 0; - } else if (property.type.equals("EXT-X-STREAM-INF")) { - if (property.properties.get("BANDWIDTH") != null) - newUrl = Long.parseLong(property.properties.get("BANDWIDTH")); - else newUrl = 1; - } else if (property.type.equals("FILE") && newUrl > 0) { - urls.add(new java.util.AbstractMap.SimpleEntry<>(newUrl, new URL(m3u8Url, line))); - newUrl = 0; - } else { - newUrl = 0; - } - } - Collections.sort(urls, new Comparator<>() { - @Override - public int compare(Map.Entry o1, Map.Entry o2) { - return -Long.compare(o1.getKey(), o2.getKey()); - } - }); - if (urls.size() > 0) { - m3u8Url = urls.get(0).getValue(); - } - } - this.callback.onProgress(5); - mediaURL = m3u8Url; - ArrayList tsDownloads = new ArrayList<>(); - var iis = getStream(client, mediaURL); - if (iis == null) { - return false; - } - try (BufferedReader br = new BufferedReader(new InputStreamReader(iis))) { - var type = KeyType.NONE; - var key = new byte[16]; - var iv = new byte[16]; - String line; - while ((line = br.readLine()) != null) { - line = line.trim(); - var property = Property.checkProperty(line); - if (property == null) - continue; - else if (property.type.equals("EXT-X-KEY")) { - if ("NONE".equals(property.properties.get("METHOD"))) { - type = KeyType.NONE; - continue; - } - type = KeyType.AES128; - var keyUrl = new URL(mediaURL, property.properties.get("URI")); - var keyStream = getStream(client, keyUrl); - if (keyStream == null) { - return false; - } - try (InputStream i = keyStream) { - var keyLength = is.read(key); - if (keyLength != key.length) { - throw new RuntimeException("wtf"); - } - } - if (property.properties.get("IV") != null) { - String ivstr = property.properties.get("IV"); - ivstr = ivstr.substring(2); - for (int i = 0; i < iv.length; i++) { - iv[i] = (byte) Integer.parseInt(ivstr.substring(i * 2, (i + 1) * 2), 16); - } - } else { - Arrays.fill(iv, (byte) 0); - } - } else if (property.type.equals("FILE")) { - var url = new URL(mediaURL, line); - tsDownloads.add(new TSDownload(url, type, key, iv)); - - for (var i = iv.length; i > 0; i--) { - iv[i - 1] = (byte) (iv[i - 1] + 1); - if (iv[i - 1] != 0) { - break; - } - } - } - } - } - this.callback.onProgress(10); - - try (var fileStream = output) { - for (var download : tsDownloads) { - if (download.start(client)) { - fileStream.write(download.getData()); - this.callback.onSizeReceived((long) download.getData().length * tsDownloads.size(), headerSize); - this.callback.onProgress(10 + Math.round(80.0f * tsDownloads.indexOf(download) / tsDownloads.size())); - if (this.returnChunk == tsDownloads.indexOf(download)) { - return true; - } else { - return false; - } - } - } - } - } - } catch (Exception e) { - Log.e("VTosters Lite", e.toString()); - return false; - } - return true; - } -} diff --git a/app/src/main/java/ru/vtosters/lite/downloaders/music/M3UDownloader.java b/app/src/main/java/ru/vtosters/lite/downloaders/music/M3UDownloader.java new file mode 100644 index 000000000..c264f9adb --- /dev/null +++ b/app/src/main/java/ru/vtosters/lite/downloaders/music/M3UDownloader.java @@ -0,0 +1,55 @@ +package ru.vtosters.lite.downloaders.music; + +import android.util.Log; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.List; + +import javax.crypto.NoSuchPaddingException; + +import ru.vtosters.lite.utils.StreamUtils; +import ru.vtosters.lite.music.TransportStream; +import ru.vtosters.lite.music.VKM3UParser; + +public class M3UDownloader { + + private String mURL; + private File mOutDir; + private Callback mCallback; + + public M3UDownloader(String url, File outDir, Callback callback) { + mURL = url; + mOutDir = outDir; + mCallback = callback; + } + + public void execute() { + VKM3UParser parser = new VKM3UParser(mURL); + List tses = parser.getTransportStreams(); + mCallback.onProgress(5); + for (TransportStream ts : tses) { + String tsURL = parser.getBaseUrl() + ts.getName(); + Log.d("M3UDownloader", "Downloading " + tsURL); + try { + InputStream is = StreamUtils.openStream(tsURL); + byte[] content = TransportStream.METHOD_AES128.equals(ts.getMethod()) + ? StreamUtils.decodeStream(is, ts.getKeyURL()) + : StreamUtils.readAllBytes(is); + File tsDump = new File(mOutDir, ts.getName()); + StreamUtils.writeToFile(tsDump, content); + mCallback.onSizeReceived((long) content.length * tses.size(), parser.getHeapSize()); + mCallback.onProgress(10 + Math.round(80.0f * tses.indexOf(ts) / tses.size())); + } catch (IOException | NoSuchPaddingException | NoSuchAlgorithmException | InvalidAlgorithmParameterException | InvalidKeyException e) { + e.printStackTrace(); + mCallback.onFailure(); + return; + } + } + mCallback.onSuccess(); + } +} diff --git a/app/src/main/java/ru/vtosters/lite/downloaders/music/Property.java b/app/src/main/java/ru/vtosters/lite/downloaders/music/Property.java deleted file mode 100644 index 31be17f72..000000000 --- a/app/src/main/java/ru/vtosters/lite/downloaders/music/Property.java +++ /dev/null @@ -1,140 +0,0 @@ -package ru.vtosters.lite.downloaders.music; - -import androidx.annotation.NonNull; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; - -public class Property { - public String type; - public Map properties; - public String[] values; - public String value; - - @NonNull - public String toString() { - StringBuilder sb = new StringBuilder("Property type = "); - sb.append(type); - if (properties != null) { - sb.append(", properties = { "); - for (Map.Entry i : properties.entrySet()) { - sb.append(i.getKey()); - sb.append(" = "); - sb.append(i.getValue()); - sb.append(", "); - } - sb.append("}"); - } - if (values != null) { - sb.append(", values = [ "); - for (String i : values) { - sb.append(i); - sb.append(", "); - } - sb.append("]"); - } - return sb.toString(); - } - - public static Property checkProperty(String line) { - Property property = parseLine(line); - if (property.type.equals("FILE")) return property; - if (property.type.equals("EXT-X-STREAM-INF") && property.properties != null) { - if (property.properties.get("BANDWIDTH") != null) { - try { - Long.parseLong(property.properties.get("BANDWIDTH")); - } catch (NumberFormatException e) { - return null; - } - } - return property; - } - - if (property.type.equals("EXTINF") && property.values != null) { - // #EXTINF:10.23, - try { - Double.parseDouble(property.values[0]); - } catch (NumberFormatException e) { - return null; - } - return property; - } - - if (property.type.equals("EXT-X-KEY") && property.properties != null) { - // #EXT-X-KEY:METHOD=AES-128,URI=\"(.*)\",IV=0[xX](.{32}) - boolean aes = "AES-128".equals(property.properties.get("METHOD")); - if (!aes && !"NONE".equals(property.properties.get("METHOD"))) { - return null; - } - if (!property.properties.containsKey("URI") && aes) { - return null; - } - if (property.properties.containsKey("IV") && !property.properties.get("IV").matches("0[xX](.{32})")) { - return null; - } - return property; - } - - return null; - } - - public static Property parseLine(String line) { - Property ret = new Property(); - if (!line.startsWith("#")) { - ret.type = "FILE"; - ret.value = line; - return ret; - } - String[] types = line.split(":", 2); - ret.type = types[0].substring(1); - if (types.length == 1) { - return ret; - } - ret.value = types[1]; - ret.values = splitProperty(types[1]); - ret.properties = new HashMap<>(); - for (String p : ret.values) { - if (p.length() > 0) { - String[] ps = p.split("=", 2); - if (ps.length == 1) { - ret.properties.put(p, null); - } else { - String k = ps[0]; - String v = ps[1]; - if (v.charAt(0) == '"' && v.charAt(v.length() - 1) == '"') { - v = v.substring(1, v.length() - 1); - } - ret.properties.put(k, v); - } - } - } - return ret; - } - - public static String[] splitProperty(String line) { - ArrayList list = new ArrayList<>(); - boolean escape = false; - boolean quote = false; - StringBuilder sb = new StringBuilder(); - - for (char c : line.toCharArray()) { - if (escape) { - escape = false; - } else if (quote) { - if (c == '"') quote = false; - } else { - if (c == ',') { - list.add(sb.toString()); - sb.delete(0, sb.length()); - continue; - } - if (c == '\\') escape = true; - if (c == '"') quote = true; - } - sb.append(c); - } - list.add(sb.toString()); - return list.toArray(new String[]{}); - } -} diff --git a/app/src/main/java/ru/vtosters/lite/downloaders/music/TSDownload.java b/app/src/main/java/ru/vtosters/lite/downloaders/music/TSDownload.java deleted file mode 100644 index 34ec7d150..000000000 --- a/app/src/main/java/ru/vtosters/lite/downloaders/music/TSDownload.java +++ /dev/null @@ -1,112 +0,0 @@ -package ru.vtosters.lite.downloaders.music; - -import android.util.Log; - -import java.io.Closeable; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.security.InvalidAlgorithmParameterException; -import java.security.InvalidKeyException; -import java.security.Key; -import java.security.NoSuchAlgorithmException; - -import javax.crypto.Cipher; -import javax.crypto.CipherInputStream; -import javax.crypto.NoSuchPaddingException; -import javax.crypto.spec.IvParameterSpec; -import javax.crypto.spec.SecretKeySpec; - -import ru.vtosters.lite.downloaders.music.utils.ByteUtils; -import ru.vtosters.lite.net.NetClient; - -public class TSDownload { - private static final int TS_PACKET_SIZE = 188; - - private final URL url; - private final KeyType type; - private final byte[] key; - private final byte[] iv; - private byte[] bytes; - - - public TSDownload(URL url, KeyType type, byte[] key, byte[] iv) { - this.url = url; - this.type = type; - this.key = key.clone(); - this.iv = iv.clone(); - } - - private static void closeQuietly(Closeable closeable) { - try { - if (closeable != null) { - closeable.close(); - } - } catch (IOException ignored) { - } - } - - private static Cipher getAesCp(byte[] key, byte[] iv) throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException { - Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); - Key skey = new SecretKeySpec(key, "AES"); - IvParameterSpec param = new IvParameterSpec(iv); - cipher.init(Cipher.DECRYPT_MODE, skey, param); - return cipher; - } - - public URL getUrl() { - return url; - } - - public KeyType getType() { - return type; - } - - public byte[] getKey() { - return key; - } - - public byte[] getIv() { - return iv; - } - - public boolean start(NetClient client) { - try { - InputStream iis = M3U8.getStream(client, getUrl()); - if (iis == null) { - return false; - } - KeyType type = getType(); - byte[] bytes; - byte[] buffer = new byte[4096]; - if (type == KeyType.AES128) { - CipherInputStream stream = new CipherInputStream(iis, getAesCp(key, iv)); - bytes = ByteUtils.toByteArray(stream); - closeQuietly(stream); - } else if (type == KeyType.NONE) { - bytes = ByteUtils.toByteArray(iis); - } else { - throw new IllegalStateException(); - } - if (bytes.length % TS_PACKET_SIZE != 0) { - throw new IllegalStateException(); - } - for (int i = 0; i < bytes.length; i = i + TS_PACKET_SIZE) { - if (bytes[i] != 0x47) { - throw new IllegalStateException(); - } - } - - this.bytes = bytes; - } catch (Exception e) { - Log.w("VTosters Lite", e); - return false; - } - return true; - } - - public byte[] getData() { - return bytes; - } -} - diff --git a/app/src/main/java/ru/vtosters/lite/downloaders/music/utils/ByteUtils.java b/app/src/main/java/ru/vtosters/lite/downloaders/music/utils/ByteUtils.java deleted file mode 100644 index 711046f91..000000000 --- a/app/src/main/java/ru/vtosters/lite/downloaders/music/utils/ByteUtils.java +++ /dev/null @@ -1,18 +0,0 @@ -package ru.vtosters.lite.downloaders.music.utils; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; - -public class ByteUtils { - // copilot did something useful wow wow wow - public static byte[] toByteArray(InputStream is) throws IOException { - ByteArrayOutputStream os = new ByteArrayOutputStream(); - byte[] buffer = new byte[8192]; - int len; - while ((len = is.read(buffer)) != -1) { - os.write(buffer, 0, len); - } - return os.toByteArray(); - } -} diff --git a/app/src/main/java/ru/vtosters/lite/music/TransportStream.java b/app/src/main/java/ru/vtosters/lite/music/TransportStream.java new file mode 100644 index 000000000..85b26347b --- /dev/null +++ b/app/src/main/java/ru/vtosters/lite/music/TransportStream.java @@ -0,0 +1,36 @@ +package ru.vtosters.lite.music; + +public class TransportStream { + + public static final String METHOD_NONE = "NONE"; + public static final String METHOD_AES128 = "AES-128"; + + private String mMethod; + private String mName; + private String mKeyURL; + + public TransportStream() { + this.mMethod = METHOD_NONE; + } + + public TransportStream(String method, String keyURL) { + this.mMethod = method; + this.mKeyURL = keyURL; + } + + public void setName(String name) { + mName = name; + } + + public String getMethod() { + return mMethod; + } + + public String getName() { + return mName; + } + + public String getKeyURL() { + return mKeyURL; + } +} diff --git a/app/src/main/java/ru/vtosters/lite/music/VKM3UParser.java b/app/src/main/java/ru/vtosters/lite/music/VKM3UParser.java new file mode 100644 index 000000000..a521dd20b --- /dev/null +++ b/app/src/main/java/ru/vtosters/lite/music/VKM3UParser.java @@ -0,0 +1,65 @@ +package ru.vtosters.lite.music; + +import java.util.ArrayList; +import java.util.List; +import java.util.Scanner; + +public class VKM3UParser { + private final String mData; + private final List mTransportStreams = new ArrayList<>(); + + // maybe will be used + private int mDuration; + private int mHeapSize = 0; + private String mBaseURL; + + public VKM3UParser(String data) { + mData = data; + init(); + } + + private void init() { + int i = 0; + Scanner scanner = new Scanner(mData); + while (scanner.hasNext()) { + String line = scanner.nextLine(); + mHeapSize += line.getBytes().length; + if (line.startsWith("#EXT-X-TARGETDURATION")) { + mDuration = Integer.parseInt(getDirectiveValue(line)); + } else if (line.startsWith("#EXT-X-KEY")) { + String base = getDirectiveValue(line); + final TransportStream ts; + if (!base.startsWith("METHOD=NONE")) { + String keyURL = base.substring(base.indexOf("\"") + 1, base.lastIndexOf("\"")); + mBaseURL = keyURL.substring(0, keyURL.lastIndexOf("/") + 1); + ts = new TransportStream(TransportStream.METHOD_AES128, keyURL); + } else { + ts = new TransportStream(); + } + mTransportStreams.add(ts); + } else if (line.startsWith("#EXTINF")) { + if (mTransportStreams.size() == i) + mTransportStreams.add(new TransportStream()); + } else if (line.endsWith(".ts") || line.endsWith(".tp") || line.endsWith(".mpeg-ts") || line.endsWith(".m2ts")) { + mTransportStreams.get(i).setName(line); + ++i; + } + } + } + + private String getDirectiveValue(String directive) { + return directive.substring(directive.indexOf(":") + 1); + } + + public int getHeapSize() { + return mHeapSize; + } + + public String getBaseUrl() { + return mBaseURL; + } + + public List getTransportStreams() { + return mTransportStreams; + } +} diff --git a/app/src/main/java/ru/vtosters/lite/utils/StreamUtils.java b/app/src/main/java/ru/vtosters/lite/utils/StreamUtils.java new file mode 100644 index 000000000..5187c3377 --- /dev/null +++ b/app/src/main/java/ru/vtosters/lite/utils/StreamUtils.java @@ -0,0 +1,65 @@ +package ru.vtosters.lite.utils; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.net.URLConnection; +import java.nio.charset.StandardCharsets; +import java.security.InvalidAlgorithmParameterException; +import java.security.InvalidKeyException; +import java.security.Key; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; + +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.NoSuchPaddingException; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +public class StreamUtils { + private static final int BUFFER_SIZE = 2048; + + public static byte[] decodeStream(InputStream encIs, String keyURL) throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, IOException { + CipherInputStream cip = new CipherInputStream(encIs, getCipher(keyURL)); + return readAllBytes(cip); + } + + public static Cipher getCipher(String keyURL) throws NoSuchPaddingException, NoSuchAlgorithmException, InvalidAlgorithmParameterException, InvalidKeyException, IOException { + String key = new String(readAllBytes(openStream(keyURL))); + Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding"); + Key keySpec = new SecretKeySpec(key.getBytes(StandardCharsets.UTF_8), "AES"); + // bypassing ecb + byte[] iv = new byte[16]; + Arrays.fill(iv, (byte) 0x0); + IvParameterSpec ivps = new IvParameterSpec(iv); + cipher.init(Cipher.DECRYPT_MODE, keySpec, ivps); + return cipher; + } + + public static InputStream openStream(String url) throws IOException { + URLConnection connection = new URL(url).openConnection(); + return connection.getInputStream(); + } + + public static String readAllLines(InputStream is) throws IOException { + return new String(readAllBytes(is), "utf-8"); + } + + public static byte[] readAllBytes(InputStream is) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + byte[] buffer = new byte[BUFFER_SIZE]; + int len = 0; + while ((len = is.read(buffer)) > 0) + bos.write(buffer, 0, len); + return bos.toByteArray(); + } + + public static void writeToFile(File file, byte[] content) throws IOException { + FileOutputStream fos = new FileOutputStream(file); + fos.write(content); + } +}