I'm using this compiler-lexer script for developing utils using Compiler::Lexer.

#!/usr/bin/env perl
use strict;
use warnings;
use Compiler::Lexer;
use Getopt::Long;

my $eval;
my $p = Getopt::Long::Parser->new(
    config => \[qw(posix\_default no\_ignore\_case auto\_help)\]
);
$p->getoptions(
    'e=s'       => \\$eval,
);

if (defined $eval) {
    dump\_tokens('-e', $eval);
} elsif (@ARGV > 0) {
    for my $fname (@ARGV) {
        printf "===> %s <===\\n", $fname if @ARGV > 1;
        my $src = slurp($fname);
        dump\_tokens($fname, $src);
    }
} else {
    my $src = join('', <>);
    dump\_tokens('-', $src);
}

sub dump\_tokens {
    my ($fname, $src) = @\_;
    my $lexer = Compiler::Lexer->new($fname);
    my $tokens = $lexer->tokenize($src);
    for (@$$tokens) {
        printf "%-20s - %s\\n", $\_->name, $\_->data;
    }
}

sub slurp {
    my $fname = shift;
    open my $fh, '<', $fname
        or Carp::croak("Can't open '$fname' for reading: '$!'");
    do { local $/; <$fh> }
}