DocShards: Large File Handling

Traditional Web:
Large File (50KB) → Web Server → Browser ✅

TELA Without DocShards:
Large File (50KB) → Smart Contract ❌ (18KB limit exceeded)

TELA With DocShards:
Large File (50KB) → Split into 3 shards → 3 Smart Contracts → Reconstruct → Browser ✅

DocShards Process:
1. large-file.js (50KB) 
2. Split → large-file-1.js (17.5KB) + large-file-2.js (17.5KB) + large-file-3.js (15KB)
3. Deploy → 3 separate DOC contracts with .shard tags
4. INDEX → References all shards with .shards tag
5. Serve → Automatically reconstructs original large-file.js

// TELA constants for sharding
const SHARD_SIZE = int64(17500) // 17.5KB per shard
 
// Calculate shard requirements
func analyzeForSharding(content []byte) {
    totalShards, fileSize := tela.GetTotalShards(content)
    
    fmt.Printf("File size: %d bytes\n", fileSize)
    fmt.Printf("Shards needed: %d\n", totalShards)
    fmt.Printf("Shard size: ~%.1fKB each\n", float64(SHARD_SIZE)/1024)
}

# 1. Analyze file size
file-info large-application.js
# Output: Size: 45.2 KB ❌ Exceeds limit, needs sharding
 
# 2. Create shard files
file-shard large-application.js
# Creates: large-application-1.js, large-application-2.js, large-application-3.js
 
# 3. Verify reconstruction works
file-construct large-application-1.js
# Recreates: large-application.js
 
# 4. Deploy each shard
install-doc large-application-1.js  # dURL: large-application-1.tela.shard
install-doc large-application-2.js  # dURL: large-application-2.tela.shard  
install-doc large-application-3.js  # dURL: large-application-3.tela.shard
 
# 5. Create INDEX with .shards tag
install-index MyApp  # dURL: myapp.tela.shards
# Include all shard SCIDs in the INDEX

package main
 
import (
    "fmt"
    "os"
    "github.com/civilware/tela"
    "github.com/deroproject/derohe/walletapi"
)
 
func deployLargeFile(wallet *walletapi.Wallet_Disk, filePath string) error {
    // 1. Read file content
    content, err := os.ReadFile(filePath)
    if err != nil {
        return err
    }
    
    // 2. Check if sharding is needed
    size := tela.GetCodeSizeInKB(string(content))
    if size <= 18.0 {
        return fmt.Errorf("file doesn't need sharding (%.2fKB)", size)
    }
    
    fmt.Printf("📊 File needs sharding: %.2fKB\n", size)
    
    // 3. Create shard files
    err = tela.CreateShardFiles(filePath, "", nil)
    if err != nil {
        return fmt.Errorf("failed to create shards: %v", err)
    }
    
    // 4. Calculate shard count
    shardCount, _ := tela.GetTotalShards(content)
    fmt.Printf("📦 Created %d shard files\n", shardCount)
    
    // 5. Deploy each shard
    var shardSCIDs []string
    fileName := filepath.Base(filePath)
    
    for i := 1; i <= shardCount; i++ {
        shardName := fmt.Sprintf("%s-%d%s", 
            strings.TrimSuffix(fileName, filepath.Ext(fileName)), 
            i, 
            filepath.Ext(fileName))
        
        shardContent, err := os.ReadFile(filepath.Join(filepath.Dir(filePath), shardName))
        if err != nil {
            return err
        }
        
        // Create DOC for shard
        shardDOC := &tela.DOC{
            DocType: tela.ParseDocType(fileName), // Original file type
            Code:    string(shardContent),
            DURL:    fmt.Sprintf("%s.tela.shard", strings.TrimSuffix(shardName, filepath.Ext(shardName))),
            Headers: tela.Headers{
                NameHdr:  shardName,
                DescrHdr: fmt.Sprintf("Shard %d of %s", i, fileName),
            },
        }
        
        txid, err := tela.Installer(wallet, 2, shardDOC)
        if err != nil {
            return fmt.Errorf("failed to deploy shard %d: %v", i, err)
        }
        
        shardSCIDs = append(shardSCIDs, txid)
        fmt.Printf("✅ Shard %d deployed: %s\n", i, txid)
    }
    
    // 6. Create INDEX with .shards tag
    index := &tela.INDEX{
        DURL: fmt.Sprintf("%s.tela.shards", strings.TrimSuffix(fileName, filepath.Ext(fileName))),
        DOCs: shardSCIDs,
        Headers: tela.Headers{
            NameHdr:  fmt.Sprintf("Sharded %s", fileName),
            DescrHdr: fmt.Sprintf("Large file deployed as %d shards", shardCount),
        },
    }
    
    indexTXID, err := tela.Installer(wallet, 2, index)
    if err != nil {
        return fmt.Errorf("failed to deploy INDEX: %v", err)
    }
    
    fmt.Printf("🎉 Sharded file deployed: %s\n", indexTXID)
    return nil
}

// Intelligent sharding decision
func shouldUseSharding(filePath string) (bool, string, error) {
    content, err := os.ReadFile(filePath)
    if err != nil {
        return false, "", err
    }
    
    originalSize := tela.GetCodeSizeInKB(string(content))
    
    // Try compression first
    compressed, err := tela.Compress(content, ".gz")
    if err == nil {
        compressedSize := tela.GetCodeSizeInKB(compressed)
        
        if compressedSize <= 18.0 {
            return false, "use compression", nil
        }
        
        if compressedSize < originalSize * 0.7 { // 30%+ reduction
            return true, "compress then shard", nil
        }
    }
    
    if originalSize > 18.0 {
        return true, "shard without compression", nil
    }
    
    return false, "deploy directly", nil
}

Project Structure for Sharding:
my-large-app/
├── src/
│   ├── small-file.js (12KB) → Deploy directly
│   ├── medium-file.js (16KB) → Compress then deploy
│   └── large-file.js (45KB) → Compress then shard
├── shards/ (auto-generated)
│   ├── large-file-1.js.gz
│   ├── large-file-2.js.gz
│   └── large-file-3.js.gz
└── deployment-plan.json (tracking)

// Track complex deployments
type DeploymentPlan struct {
    ProjectName string                 `json:"projectName"`
    Files       []FileDeploymentInfo   `json:"files"`
    Shards      []ShardDeploymentInfo  `json:"shards"`
    INDEX       IndexDeploymentInfo    `json:"index"`
}
 
type FileDeploymentInfo struct {
    FileName    string `json:"fileName"`
    FilePath    string `json:"filePath"`
    Size        float64 `json:"size"`
    Compressed  bool   `json:"compressed"`
    SCID        string `json:"scid"`
    Status      string `json:"status"`
}
 
type ShardDeploymentInfo struct {
    OriginalFile string   `json:"originalFile"`
    ShardFiles   []string `json:"shardFiles"`
    ShardSCIDs   []string `json:"shardSCIDs"`
    Status       string   `json:"status"`
}
 
// Save deployment progress
func saveDeploymentPlan(plan *DeploymentPlan) error {
    data, err := json.MarshalIndent(plan, "", "  ")
    if err != nil {
        return err
    }
    
    return os.WriteFile("deployment-plan.json", data, 0644)
}

func deployMultipleShardedFiles(wallet *walletapi.Wallet_Disk, largeFiles []string) (*tela.INDEX, error) {
    var allDOCs []string
    
    for _, filePath := range largeFiles {
        fmt.Printf("Processing large file: %s\n", filePath)
        
        // Create shards
        err := tela.CreateShardFiles(filePath, ".gz", nil)
        if err != nil {
            return nil, err
        }
        
        // Deploy shards
        shardSCIDs, err := deployFileShards(wallet, filePath)
        if err != nil {
            return nil, err
        }
        
        allDOCs = append(allDOCs, shardSCIDs...)
    }
    
    // Create master INDEX
    index := &tela.INDEX{
        DURL: "large-app.tela.shards",
        DOCs: allDOCs,
        Headers: tela.Headers{
            NameHdr:  "Large Application",
            DescrHdr: fmt.Sprintf("Application with %d sharded files", len(largeFiles)),
        },
    }
    
    return index, nil
}

// Client-side: Handle sharded vs non-sharded content
class TELAContentLoader {
    async loadContent(scid) {
        const info = await this.getContentInfo(scid);
        
        if (info.dURL.endsWith('.shards')) {
            // Handle sharded content
            return await this.loadShardedContent(scid);
        } else {
            // Handle regular content
            return await this.loadRegularContent(scid);
        }
    }
    
    async loadShardedContent(scid) {
        console.log('📦 Loading sharded content...');
        
        // TELA automatically handles reconstruction
        // Your app just needs to be aware it might take longer
        return await this.fetchWithProgress(scid);
    }
    
    async fetchWithProgress(scid) {
        return new Promise((resolve, reject) => {
            // Show loading indicator for sharded content
            this.showLoadingIndicator('Reconstructing large files...');
            
            // Fetch normally - TELA handles the complexity
            fetch(`/content/${scid}`)
                .then(response => response.text())
                .then(content => {
                    this.hideLoadingIndicator();
                    resolve(content);
                })
                .catch(reject);
        });
    }
}

# Check shard integrity
file-construct large-file-1.js
# If this fails, shards may be corrupted
 
# Recreate shards
rm large-file-*.js  # Remove old shards
file-shard original-large-file.js  # Recreate

# Deploy shards in correct order
install-doc large-file-1.js  # First shard
install-doc large-file-2.js  # Second shard
install-doc large-file-3.js  # Last shard
 
# Then create INDEX
install-index MyApp  # With .shards tag

// Optimize shard deployment
func deployShardsOptimally(wallet *walletapi.Wallet_Disk, shardFiles []string) error {
    // Deploy in parallel (be careful with rate limits)
    semaphore := make(chan struct{}, 3) // Max 3 concurrent
    
    for _, shardFile := range shardFiles {
        go func(file string) {
            semaphore <- struct{}{} // Acquire
            defer func() { <-semaphore }() // Release
            
            // Deploy shard
            deployShardFile(wallet, file)
        }(shardFile)
    }
    
    return nil
}

// Smart content-aware sharding
func intelligentSharding(content []byte, contentType string) ([][]byte, error) {
    switch contentType {
    case "TELA-JS-1":
        // Split JavaScript at function boundaries
        return splitJavaScriptIntelligently(content)
    case "TELA-CSS-1":
        // Split CSS at rule boundaries
        return splitCSSIntelligently(content)
    case "TELA-HTML-1":
        // Split HTML at logical sections
        return splitHTMLIntelligently(content)
    default:
        // Use standard byte-based splitting
        return splitByBytes(content)
    }
}
 
func splitJavaScriptIntelligently(content []byte) ([][]byte, error) {
    jsCode := string(content)
    
    // Find function boundaries
    functions := extractJSFunctions(jsCode)
    
    var shards [][]byte
    var currentShard []byte
    
    for _, function := range functions {
        functionBytes := []byte(function)
        
        // If adding this function would exceed shard size, start new shard
        if len(currentShard)+len(functionBytes) > int(SHARD_SIZE) {
            if len(currentShard) > 0 {
                shards = append(shards, currentShard)
                currentShard = nil
            }
        }
        
        currentShard = append(currentShard, functionBytes...)
    }
    
    // Add final shard
    if len(currentShard) > 0 {
        shards = append(shards, currentShard)
    }
    
    return shards, nil
}

func optimizedSharding(filePath string) error {
    content, err := os.ReadFile(filePath)
    if err != nil {
        return err
    }
    
    originalSize := tela.GetCodeSizeInKB(string(content))
    fmt.Printf("Original size: %.2fKB\n", originalSize)
    
    // Try compression first
    compressed, err := tela.Compress(content, ".gz")
    if err != nil {
        return err
    }
    
    compressedSize := tela.GetCodeSizeInKB(compressed)
    fmt.Printf("Compressed size: %.2fKB\n", compressedSize)
    
    if compressedSize <= 18.0 {
        fmt.Println("✅ Compression sufficient, no sharding needed")
        return nil
    }
    
    // Create compressed shards
    err = tela.CreateShardFiles(filePath, ".gz", []byte(compressed))
    if err != nil {
        return err
    }
    
    shardCount, _ := tela.GetTotalShards([]byte(compressed))
    fmt.Printf("🧩 Created %d compressed shards\n", shardCount)
    
    return nil
}

// Example: Deploying React or Vue.js to TELA
func deployJSFramework(wallet *walletapi.Wallet_Disk, frameworkPath string) error {
    // 1. Analyze framework size
    content, err := os.ReadFile(frameworkPath)
    if err != nil {
        return err
    }
    
    size := tela.GetCodeSizeInKB(string(content))
    fmt.Printf("📊 Framework size: %.2fKB\n", size)
    
    if size <= 18.0 {
        // Small framework, deploy directly
        return deployRegularDOC(wallet, frameworkPath)
    }
    
    // 2. Create optimized shards
    // First try compression
    compressed, err := tela.Compress(content, ".gz")
    if err != nil {
        return err
    }
    
    // Create shards from compressed content
    err = tela.CreateShardFiles(frameworkPath, ".gz", []byte(compressed))
    if err != nil {
        return err
    }
    
    // 3. Deploy framework shards
    shardCount, _ := tela.GetTotalShards([]byte(compressed))
    var shardSCIDs []string
    
    for i := 1; i <= shardCount; i++ {
        shardSCID, err := deployFrameworkShard(wallet, frameworkPath, i)
        if err != nil {
            return err
        }
        shardSCIDs = append(shardSCIDs, shardSCID)
    }
    
    // 4. Create framework INDEX
    frameworkIndex := &tela.INDEX{
        DURL: "react.tela.shards", // or vue.tela.shards
        DOCs: shardSCIDs,
        Headers: tela.Headers{
            NameHdr:  "React Framework",
            DescrHdr: "React.js deployed as DocShards",
        },
    }
    
    indexTXID, err := tela.Installer(wallet, 2, frameworkIndex)
    if err != nil {
        return err
    }
    
    fmt.Printf("🎉 Framework deployed: %s\n", indexTXID)
    return nil
}

// Example: Deploying large JSON datasets
func deployLargeDataset(wallet *walletapi.Wallet_Disk, dataPath string) error {
    // 1. Load and validate JSON
    content, err := os.ReadFile(dataPath)
    if err != nil {
        return err
    }
    
    // Validate JSON
    var jsonData interface{}
    if err := json.Unmarshal(content, &jsonData); err != nil {
        return fmt.Errorf("invalid JSON: %v", err)
    }
    
    size := tela.GetCodeSizeInKB(string(content))
    fmt.Printf("📊 Dataset size: %.2fKB\n", size)
    
    if size <= 18.0 {
        return deployRegularDOC(wallet, dataPath)
    }
    
    // 2. Compress JSON (usually very effective)
    compressed, err := tela.Compress(content, ".gz")
    if err != nil {
        return err
    }
    
    compressedSize := tela.GetCodeSizeInKB(compressed)
    fmt.Printf("📦 Compressed to: %.2fKB (%.1f%% reduction)\n", 
        compressedSize, (1-compressedSize/size)*100)
    
    if compressedSize <= 18.0 {
        // Compression sufficient
        doc := &tela.DOC{
            DocType:     tela.DOC_JSON,
            Code:        compressed,
            DURL:        "dataset.tela",
            Compression: ".gz",
            Headers: tela.Headers{
                NameHdr:  filepath.Base(dataPath) + ".gz",
                DescrHdr: "Compressed JSON dataset",
            },
        }
        
        txid, err := tela.Installer(wallet, 2, doc)
        if err != nil {
            return err
        }
        
        fmt.Printf("✅ Compressed dataset deployed: %s\n", txid)
        return nil
    }
    
    // 3. Need sharding even with compression
    err = tela.CreateShardFiles(dataPath, ".gz", []byte(compressed))
    if err != nil {
        return err
    }
    
    // Deploy shards and INDEX
    return deployDatasetShards(wallet, dataPath)
}

// Client-side: Optimize sharded content loading
class ShardedContentManager {
    constructor() {
        this.loadingCache = new Map();
        this.reconstructionWorkers = new Map();
    }
    
    async loadShardedContent(scid) {
        // Check if already loading
        if (this.loadingCache.has(scid)) {
            return await this.loadingCache.get(scid);
        }
        
        // Create loading promise
        const loadingPromise = this.performShardedLoad(scid);
        this.loadingCache.set(scid, loadingPromise);
        
        try {
            const result = await loadingPromise;
            return result;
        } finally {
            this.loadingCache.delete(scid);
        }
    }
    
    async performShardedLoad(scid) {
        // Show progress for large reconstructions
        this.showProgress('Loading sharded content...', 0);
        
        // TELA handles the actual reconstruction
        // We just provide user feedback
        const startTime = Date.now();
        
        try {
            const content = await fetch(`/content/${scid}`).then(r => r.text());
            
            const loadTime = Date.now() - startTime;
            console.log(`📦 Sharded content loaded in ${loadTime}ms`);
            
            this.hideProgress();
            return content;
        } catch (error) {
            this.hideProgress();
            throw error;
        }
    }
    
    showProgress(message, percent) {
        // Implement progress UI
        const progressEl = document.getElementById('loading-progress');
        if (progressEl) {
            progressEl.textContent = `${message} ${percent}%`;
            progressEl.style.display = 'block';
        }
    }
    
    hideProgress() {
        const progressEl = document.getElementById('loading-progress');
        if (progressEl) {
            progressEl.style.display = 'none';
        }
    }
}