The Perl and Raku Conference 2025: Greenville, South Carolina - June 27-29 Learn more

VVU / wubot-0.3.1-TRIAL / lib / App / Wubot / Reactor / WebFetch.pm 

            

              
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
—
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

              
package App::Wubot::Reactor::WebFetch;
use Moose;
our $VERSION = '0.3.1'; # VERSION
use YAML;
use App::Wubot::Logger;
use App::Wubot::Util::WebFetcher;
has 'fetcher' => ( is  => 'ro',
                   isa => 'App::Wubot::Util::WebFetcher',
                   lazy => 1,
                   default => sub {
                       return App::Wubot::Util::WebFetcher->new();
                   },
               );
has 'logger'  => ( is => 'ro',
                   isa => 'Log::Log4perl::Logger',
                   lazy => 1,
                   default => sub {
                       return Log::Log4perl::get_logger( __PACKAGE__ );
                   },
               );
sub react {
    my ( $self, $message, $config ) = @_;
    my $url;
    if ( $config->{url} ) {
        $url = $config->{url};
    }
    elsif ( $config->{url_field} ) {
        if ( $message->{ $config->{url_field} } ) {
            $url = $message->{ $config->{url_field} };
        }
        else {
            $self->logger->error( "Waring: url field not found on message: $config->{url_field}" );
            return $message;
        }
    }
    else {
        $self->logger->error( "WebFetch Reactor ERROR:: neither url nor url_field defined in config" );
        return $message;
    }
    $self->logger->debug( "Fetching content from $url" );
    my $content;
    eval {                          # try
        $content = $self->fetcher->fetch( $url, $config );
        1;
    } or do {                       # catch
        my $error = $@;
        $self->logger->error( $self->key . ": Request failure: $error" );
        return $message;
    };
    if ( $config->{field} ) {
        $message->{ $config->{field} } = $content;
        utf8::decode( $message->{ $config->{field} } );
    }
    else {
        $self->logger->error( "WebFetch Reactor ERROR: 'field' not defined in config!" );
        return $message;
    }
    return $message;
}
__PACKAGE__->meta->make_immutable;
1;
__END__
=head1 NAME
App::Wubot::Reactor::WebFetch - fetch data from a URL
=head1 VERSION
version 0.3.1
=head1 SYNOPSIS
    - name: fetch 'body' field from the 'link' field
      plugin: WebFetch
      config:
        field: body
        url_field: link
=head1 DESCRIPTION
A reactor plugin that can fetch data from a URL and store the
retrieved content on a field on the message.
This plugin is great for fetching the complete article for the body of
an RSS feed when the feed only provides a summary of the article.
The 'url' may be defined in the config, or else a 'url_field' may be
configured so that the link can be pulled from the message.  The
retrieved content is stored on the message in the configured 'field'.
If an error occurs, the error will be logged at 'error' level.
The retrieved content will be utf8 decoded.
=head1 SUBROUTINES/METHODS
=over 8
=item react( $message, $config )
The standard reactor plugin react() method.
=back