This analyzer shows how much data has been uploaded (uploaded to the storage node, i.e. downloaded from the internet, i.e. ingress traffic) and produces a histogram and total counts from it.

#!/usr/bin/env python3
import sys
import json

def human_readable_size(size_bytes):
    if size_bytes == 0:
        return "0B"
    size_name = ("B", "KB", "MB", "GB", "TB")
    i = int(len(str(size_bytes)) - 1) // 3
    p = pow(1024, i)
    s = round(size_bytes / p, 3)
    return f"{s} {size_name[i]}"

def print_histogram(sizes):
    bins = [
        (1 * 1024, 2 * 1024),
        (2 * 1024, 4 * 1024),
        (4 * 1024, 8 * 1024),
        (8 * 1024, 16 * 1024),
        (16 * 1024, 32 * 1024),
        (32 * 1024, 64 * 1024),
        (64 * 1024, 128 * 1024),
        (128 * 1024, 256 * 1024),
        (256 * 1024, 512 * 1024),
        (512 * 1024, 1 * 1024 * 1024),
        (1 * 1024 * 1024, 2 * 1024 * 1024),
        (2 * 1024 * 1024, 4 * 1024 * 1024),
        (4 * 1024 * 1024, 8 * 1024 * 1024),
        (8 * 1024 * 1024, 16 * 1024 * 1024),
        (16 * 1024 * 1024, 32 * 1024 * 1024),
        (32 * 1024, 64 * 1024),
        (64 * 1024 * 1024, 128 * 1024 * 1024),
        (128 * 1024 * 1024, 256 * 1024 * 1024),
        (256 * 1024 * 1024, float('inf'))
    ]
    bin_counts = [0] * len(bins)

    for size in sizes:
        for i, (low, high) in enumerate(bins):
            if low <= size < high:
                bin_counts[i] += 1
                break

    max_count = max(bin_counts)
    max_chars = 50

    print("File size histogram:")
    for i, (low, high) in enumerate(bins):
        bin_label = f"{human_readable_size(low)}..{human_readable_size(high)}"
        num_chars = int((bin_counts[i] / max_count) * max_chars) if max_count > 0 else 0
        count_str = f"{bin_counts[i]:8}"
        print(f"{bin_label:<20} | {count_str} | {'█' * num_chars}")

def filter_uploaded_lines(logfile):
    total_size = 0
    upload_count = 0
    sizes = []

    try:
        with open(logfile, 'r') as file:
            lines = file.readlines()

        for line in lines:
            if "uploaded" in line:
                upload_count += 1
                try:
                    json_part = line.split('uploaded')[-1].strip()
                    data = json.loads(json_part)
                    size = data.get("Size", 0)
                    total_size += size
                    sizes.append(size)
                except json.JSONDecodeError:
                    print(f"Failed to decode JSON from line: {line}")
                except Exception as e:
                    print(f"An error occurred while processing line: {e}")

        print(f"Total size of uploaded pieces: {human_readable_size(total_size)}")
        print(f"Total number of uploads: {upload_count}")
        print_histogram(sizes)

    except FileNotFoundError:
        print(f"The file {logfile} does not exist.")
    except Exception as e:
        print(f"An error occurred: {e}")

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python filter_log.py <logfile>")
    else:
        logfile = sys.argv[1]
        filter_uploaded_lines(logfile)

Run it on the storj logfile:

python3 storj-stats.py storage/storj.log

Example output

Total size of uploaded pieces: 156.995 GB
Total number of uploads: 1036601
File size histogram:
1.0 KB..2.0 KB       |    20164 | ██
2.0 KB..4.0 KB       |   402924 | █████████████████████████████████████████
4.0 KB..8.0 KB       |    18736 | █
8.0 KB..16.0 KB      |    15639 | █
16.0 KB..32.0 KB     |    10104 | █
32.0 KB..64.0 KB     |    11642 | █
64.0 KB..128.0 KB    |     9830 | █
128.0 KB..256.0 KB   |   483561 | ██████████████████████████████████████████████████
256.0 KB..512.0 KB   |    30104 | ███
512.0 KB..1.0 MB     |     3835 | 
1.0 MB..2.0 MB       |    11190 | █
2.0 MB..4.0 MB       |     8296 | 
4.0 MB..8.0 MB       |        5 | 
8.0 MB..16.0 MB      |        0 | 
16.0 MB..32.0 MB     |        0 | 
32.0 KB..64.0 KB     |        0 | 
64.0 MB..128.0 MB    |        0 | 
128.0 MB..256.0 MB   |        0 | 
256.0 MB..inf B      |        0 | 

#!/usr/bin/env python3
import sys
from collections import Counter

def main():
    if len(sys.argv) != 2:
        print("Usage: python count_lines.py <filename>")
        sys.exit(1)

    filename = sys.argv[1]
    
    try:
        with open(filename, 'r') as file:
            lines = file.readlines()
    except FileNotFoundError:
        print(f"File '{filename}' not found.")
        sys.exit(1)
    
    # Count occurrences of each line
    line_counts = Counter(lines)
    
    # Create a sorted list of lines that occur more than once
    sorted_lines = sorted(
        [(count, line.strip()) for line, count in line_counts.items() if count > 1],
        reverse=False,
        key=lambda x: x[0]
    )
    
    # Print the sorted lines
    for count, line in sorted_lines:
        print(f"{count} {line}")

if __name__ == "__main__":
    main()

This script prints file- and directorynames recursively which have characters in their name not belonging to either the Unicode basic latin or Latin-1 supplement blocks.

#!/usr/bin/env python3
import os
import argparse

def is_valid_char(c):
    """Check if a character is within Basic Latin or Latin-1 Supplement Unicode blocks."""
    return '0000' <= f"{ord(c):04x}" <= '00ff'

def contains_invalid_chars(name):
    """Check if a string contains characters outside Basic Latin or Latin-1 Supplement."""
    return any(not is_valid_char(c) for c in name)

def check_directory(path):
    """Recursively check directories and files for invalid characters in their names."""
    for root, dirs, files in os.walk(path):
        for name in dirs + files:
            if contains_invalid_chars(name):
                print(os.path.join(root, name))

def main():
    parser = argparse.ArgumentParser(description="Check for files and directories with invalid characters in their names.")
    parser.add_argument('path', type=str, help="The path to the directory to check.")
    
    args = parser.parse_args()
    
    if not os.path.isdir(args.path):
        print(f"The path {args.path} is not a valid directory.")
        return

    check_directory(args.path)

if __name__ == "__main__":
    main()

#!/usr/bin/env python3
import os
import sys

# Check if the directory is provided as a command line argument
if len(sys.argv) != 2:
    print("Usage: python merge_vcf.py <directory>")
    sys.exit(1)

# Get the directory from the command line argument
directory = sys.argv[1]

# Check if the provided argument is a valid directory
if not os.path.isdir(directory):
    print(f"{directory} is not a valid directory.")
    sys.exit(1)

# Set the output file name to the directory name plus ".vcf"
output_file = os.path.basename(os.path.normpath(directory)) + '.vcf'
output_file_path = output_file

# Check if the output file already exists
if os.path.exists(output_file_path):
    print(f"{output_file_path} already exists. Exiting...")
else:
    # Get a list of all .vcf files in the specified directory
    vcf_files = [file for file in os.listdir(directory) if file.endswith('.vcf') and file != output_file]

    # Open the output file in write mode
    with open(output_file_path, 'w', encoding="utf-8") as outfile:
        # Loop through each vcf file and append its contents to the output file
        for vcf in vcf_files:
            with open(os.path.join(directory, vcf), 'r', encoding="utf-8") as infile:
                try:
                    outfile.write(infile.read())
                    # Ensure there's a new line between different vCard files
                    outfile.write('\n')
                except UnicodeEncodeError:
                    print("Encode error while processing", vcf)

    print(f"All vCard files have been merged into {output_file_path}")

If you’ve got a CSV with a header such as

"Email Address","First Name","Last Name",MEMBER_RATING,OPTIN_TIME,OPTIN_IP,CONFIRM_TIME,CONFIRM_IP,LATITUDE,LONGITUDE,GMTOFF,DSTOFF,TIMEZONE,CC,REGION,LAST_CHANGED,LEID,EUID,NOTES

you can use the following Python script to convert it to a directory of vcard files:

#!/usr/bin/env python3
import csv
import os
import argparse

# Set up argument parser
parser = argparse.ArgumentParser(description='Convert a CSV file of contacts to vCard files.')
parser.add_argument('csv_file', type=str, help='Path to the CSV file containing contacts')

args = parser.parse_args()

# Define the path for the CSV file from command line argument
csv_file_path = args.csv_file

# Create a directory to store the vCard files
vcard_directory = 'vcards'
os.makedirs(vcard_directory, exist_ok=True)

# Read the CSV file and process each contact
with open(csv_file_path, mode='r', encoding='utf-8') as csv_file:
    csv_reader = csv.DictReader(csv_file)
    
    for row in csv_reader:
        # Extract contact details
        email = row['Email Address']
        first_name = row['First Name']
        last_name = row['Last Name']
        latitude = row['LATITUDE']
        longitude = row['LONGITUDE']
        notes = row['NOTES']
        
        # Create the vCard content
        vcard_content = (
            f"BEGIN:VCARD\n"
            f"VERSION:3.0\n"
            f"N:{last_name};{first_name};;;\n"
            f"FN:{first_name} {last_name}\n"
            f"EMAIL;TYPE=INTERNET:{email}\n"
        )
        
        if latitude and longitude:
            vcard_content += f"GEO:{latitude};{longitude}\n"
        
        if notes:
            vcard_content += f"NOTE:{notes}\n"
        
        vcard_content += "END:VCARD\n"
        
        # Define the file name for the vCard
        vcard_file_name = f"{first_name}_{last_name}.vcf".replace(" ", "_")
        vcard_file_path = os.path.join(vcard_directory, vcard_file_name)
        
        # Write the vCard content to the file
        with open(vcard_file_path, mode='w', encoding='utf-8') as vcard_file:
            vcard_file.write(vcard_content)

print(f"vCard files have been created in the '{vcard_directory}' directory.")

First, download this tool:

wget https://raw.githubusercontent.com/cclgroupltd/ccl-bplist/master/ccl_bplist.py

Now create bplist2json.py which is based on this old gist by Benno Kruit

#!/usr/bin/env python3
"""
Convert an Apple Binary Property List (bplist) to json
"""

import ccl_bplist # https://github.com/cclgroupltd/ccl-bplist

from datetime import datetime

def clean_archive(d):
    if type(d) in [dict, ccl_bplist.NsKeyedArchiverDictionary]:
        return {k:clean_archive(v) for k,v in d.items() if not k.startswith('$')}
    elif type(d) == ccl_bplist.NsKeyedArchiverList:
        return [clean_archive(i) for i in d]
    else:
        return d

def bplist_dict(fobj):
    """Convert a bplist file object to python dict"""
    plist = ccl_bplist.load(fobj)
    ccl_bplist.set_object_converter(ccl_bplist.NSKeyedArchiver_common_objects_convertor)
    archive = ccl_bplist.deserialise_NsKeyedArchiver(plist)
    return clean_archive(archive)


if __name__ == '__main__':
    import argparse, sys, json
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--input', '-i', nargs='?', type=argparse.FileType('rb'),
        default=sys.stdin, help='default: stdin')
    args = parser.parse_args()

    class ExportEncoder(json.JSONEncoder):
        def default(self, o):
            if isinstance(o, datetime):
                return o.isoformat()
            if isinstance(o, bytes):
                return o.decode('iso-8859-1')
            return json.JSONEncoder.default(self, o)
    
    print(json.dumps(bplist_dict(args.input), cls=ExportEncoder, indent=4))

Now create my script extract-chatlog.py which converts the JSON files to human readable chat logs

#!/usr/bin/env python3
import json
import sys
from datetime import datetime

# Function to format datetime string
def format_datetime(dt_str):
    dt = datetime.fromisoformat(dt_str)
    return dt.strftime("%Y-%m-%d %H:%M:%S")

def main(input_file):
    # Read JSON data from input file
    with open(input_file, 'r', encoding='utf-8') as f:
        data = json.load(f)

    # Extract the messages list
    messages = data[2]

    # Extract chat log in a human-readable format
    chat_log = []
    for message in messages:
        sender = message['Sender']['ID']
        time = format_datetime(message['Time'])
        text = message['MessageText']['NSString']
        chat_log.append(f"{time} - {sender}: {text}")

    # Print chat log to stdout
    for log in chat_log:
        print(log)

if __name__ == "__main__":
    if len(sys.argv) != 2:
        print("Usage: python script.py <input_file>")
        sys.exit(1)

    input_file = sys.argv[1]
    main(input_file)

Now it’s time to run it. Generally, run bplist2json.py on the .ichat to obtain a JSON file and then run extract-chatlog.py to obtain a text conversation log with timestamps etc.

Here’s a bash oneliner to run it on all *.ichat files in the current directory

for i in *.ichat ; do ./bplist2json.py -i ${i} > "${i}.json" ; ./extract-chatlog.py "${i}.json" > ${i}.txt ; done