CodeOverflow

Great question! Here are powerful techniques for debugging production issues without downtime:

1. Dynamic Logging with Bytecode Instrumentation

Use BTrace or Arthas to add logging dynamically:

// BTrace script to log method parameters
import com.sun.btrace.annotations.*;
import static com.sun.btrace.BTraceUtils.*;

@BTrace
public class UserServiceTracer {
    
    @OnMethod(
        clazz="com.example.service.UserService",
        method="processEmail"
    )
    public static void onProcessEmail(@Self Object self, AnyType user) {
        println("=== processEmail called ===");
        println(strcat("User object: ", str(user)));
        println(strcat("User.getEmail(): ", str(get(field(classOf(user), "email"), user))));
        println("===========================");
    }
}

Or use Arthas (easier and more powerful):

# Attach to running JVM
java -jar arthas-boot.jar

# Watch method invocations with parameters and return values
watch com.example.service.UserService processEmail "{params,returnObj,throwExp}" -x 3

# Monitor when user.email is null
watch com.example.service.UserService processEmail "{params[0].email}" "params[0].email==null" -x 3

# Decompile to see actual runtime code
jad com.example.service.UserService processEmail

2. Add Conditional Breakpoints with Remote Debugging

# Enable remote debugging in production (use SSH tunnel for security)
java -agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5005 \
     -jar application.jar

# In IntelliJ, set conditional breakpoint:
# Condition: user == null || user.getEmail() == null

3. Enhanced Logging Strategy

@Service
public class UserService {
    
    private static final Logger log = LoggerFactory.getLogger(UserService.class);
    
    public void processEmail(User user) {
        // Defensive logging
        if (user == null) {
            log.error("processEmail called with null user - this should never happen!");
            return;
        }
        
        String userEmail = user.getEmail();
        
        if (userEmail == null) {
            log.error("User {} has null email. User details: {}", 
                     user.getId(), 
                     user.toString(),
                     new Exception("Stack trace for null email"));
            return;
        }
        
        if (userEmail.length() > 0) {
            log.debug("Sending notification to {}", userEmail);
            sendNotification(userEmail);
        }
    }
}

4. Thread Dump Analysis

# Capture thread dump when issue occurs
jstack -l [PID] > thread_dump_$(date +%Y%m%d_%H%M%S).txt

# Or use jcmd (recommended)
jcmd [PID] Thread.print > thread_dump.txt

# Automated thread dump on high CPU
while true; do
    CPU=$(top -b -n 1 | grep java | awk '{print $9}')
    if (( $(echo "$CPU > 80" | bc -l) )); then
        jstack [PID] > "dump_$(date +%s).txt"
    fi
    sleep 10
done

5. Flight Recorder for Detailed Events

# Start recording when issue is suspected
jcmd [PID] JFR.start name=debug duration=60s filename=/tmp/recording.jfr

# Dump current recording
jcmd [PID] JFR.dump name=debug filename=/tmp/dump.jfr

# Analyze with JDK Mission Control or IntelliJ

6. Custom Exception Handler with Context

@ControllerAdvice
public class GlobalExceptionHandler {
    
    private static final Logger log = LoggerFactory.getLogger(GlobalExceptionHandler.class);
    
    @ExceptionHandler(NullPointerException.class)
    public ResponseEntity handleNPE(
            NullPointerException ex, 
            HttpServletRequest request) {
        
        // Capture full context
        Map context = new HashMap<>();
        context.put("url", request.getRequestURL().toString());
        context.put("method", request.getMethod());
        context.put("headers", Collections.list(request.getHeaderNames()));
        context.put("parameters", request.getParameterMap());
        context.put("sessionId", request.getSession(false) != null ? 
                                request.getSession().getId() : "none");
        context.put("user", SecurityContextHolder.getContext()
                           .getAuthentication()?.getName());
        
        log.error("NullPointerException in production. Context: {}", 
                 context, ex);
        
        // Send to monitoring system
        sendToDatadog(ex, context);
        
        return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR)
                           .body(new ErrorResponse("Internal error"));
    }
}

7. Aspect-Based Debugging

@Aspect
@Component
public class DebugAspect {
    
    private static final Logger log = LoggerFactory.getLogger(DebugAspect.class);
    
    @Around("execution(* com.example.service.*.*(..))")
    public Object logMethodExecution(ProceedingJoinPoint joinPoint) throws Throwable {
        String methodName = joinPoint.getSignature().getName();
        Object[] args = joinPoint.getArgs();
        
        log.debug("Entering: {} with args: {}", methodName, args);
        
        try {
            Object result = joinPoint.proceed();
            log.debug("Exiting: {} with result: {}", methodName, result);
            return result;
        } catch (Exception ex) {
            log.error("Exception in: {} with args: {}", methodName, args, ex);
            throw ex;
        }
    }
}

8. Memory Dump Analysis

# Capture heap dump
jmap -dump:live,format=b,file=/tmp/heap_dump.hprof [PID]

# Or use jcmd
jcmd [PID] GC.heap_dump /tmp/heap_dump.hprof

# Analyze with Eclipse MAT or VisualVM to find:
# - Objects with null fields
# - Memory leaks
# - Object retention patterns

Prevention Strategy:

// Use Optional to make nullability explicit
public void processEmail(User user) {
    Optional.ofNullable(user)
            .map(User::getEmail)
            .filter(email -> !email.isEmpty())
            .ifPresent(this::sendNotification);
}

// Or use Objects.requireNonNull for fail-fast
public void processEmail(User user) {
    Objects.requireNonNull(user, "User cannot be null");
    String email = Objects.requireNonNull(user.getEmail(), 
                                         "User email cannot be null");
    if (!email.isEmpty()) {
        sendNotification(email);
    }
}

These techniques helped me debug and fix over 50 production issues without a single restart!