summaryrefslogtreecommitdiffstats
path: root/wsutil/console_win32.c
blob: eed18c8887eef1e1ba2cf2a4efd425077ea91cb5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
/* console_win32.c
 * Console support for MSWindows
 *
 * Wireshark - Network traffic analyzer
 * By Gerald Combs <gerald@wireshark.org>
 * Copyright 2002, Jeffrey C. Foster <jfoste@woodward.com>
 *
 * SPDX-License-Identifier: GPL-2.0-or-later
 */

#ifdef _WIN32

#include <string.h>
#include <stdio.h>
#include <stdlib.h>

#include <glib.h>
#include <wsutil/file_util.h>

#include "console_win32.h"

#include <fcntl.h>
#include <conio.h>
#include <windows.h>
#include <tchar.h>

static bool has_console;  /* true if app has console */
static bool console_wait; /* "Press any key..." */
static bool stdin_capture = false; /* Don't grab stdin & stdout if true */

/*
 * Check whether a given standard handle needs to be redirected.
 *
 * If you run a Windows-subsystem program from cmd.exe on Windows XP,
 * and you haven't redirected the handle in question, GetStdHandle()
 * succeeds (so it doesn't return INVALID_HANDLE_VALUE or NULL), but
 * GetFile_type fails on the results with ERROR_INVALID_HANDLE.
 * In that case, redirection to a console is necessary.
 *
 * If you run it from the shell prompt in "mintty" in at least some
 * versions of Cygwin on Windows XP, and you haven't redirected the
 * handle in question, GetStdHandle() succeeds and returns a handle
 * that's a pipe or socket; it appears mintty reads from it and outputs
 * what it reads to the console.
 */
static bool
needs_redirection(int std_handle)
{
    HANDLE fd;
    DWORD handle_type;
    DWORD error;

    fd = GetStdHandle(std_handle);
    if (fd == NULL) {
        /*
         * No standard handle.  According to Microsoft's
         * documentation for GetStdHandle(), one reason for
         * this would be that the process is "a service on
         * an interactive desktop"; I'm not sure whether
         * such a process should be popping up a console.
         *
         * However, it also appears to be the case for
         * the standard input and standard error, but
         * *not* the standard output, for something run
         * with a double-click in Windows Explorer,
         * sow we'll say it needs redirection.
         */
        return true;
    }
    if (fd == INVALID_HANDLE_VALUE) {
        /*
         * OK, I'm not when this would happen; return
         * "no redirection" for now.
         */
        return false;
    }
    handle_type = GetFileType(fd);
    if (handle_type == FILE_TYPE_UNKNOWN) {
        error = GetLastError();
        if (error == ERROR_INVALID_HANDLE) {
            /*
             * OK, this appears to be the case where we're
             * running something in a mode that needs a
             * console.
             */
            return true;
        }
    }

    /*
     * Assume no redirection is needed for all other cases.
     */
    return false;
}

/*
 * If this application has no console window to which its standard output
 * would go, create one.
 */
void
create_console(void)
{
    bool must_redirect_stdin;
    bool must_redirect_stdout;
    bool must_redirect_stderr;

    if (stdin_capture) {
        /* We've been handed "-i -". Don't mess with stdio. */
        return;
    }

    if (has_console) {
        return;
    }

    /* Are the standard input, output, and error invalid handles? */
    must_redirect_stdin = needs_redirection(STD_INPUT_HANDLE);
    must_redirect_stdout = needs_redirection(STD_OUTPUT_HANDLE);
    must_redirect_stderr = needs_redirection(STD_ERROR_HANDLE);

    /* If none of them are invalid, we don't need to do anything. */
    if (!must_redirect_stdin && !must_redirect_stdout && !must_redirect_stderr)
        return;

    /* OK, at least one of them needs to be redirected to a console;
        try to attach to the parent process's console and, if that fails,
        try to create one. */
    /*
        * See if we have an existing console (i.e. we were run from a
        * command prompt).
        */
    if (!AttachConsole(ATTACH_PARENT_PROCESS)) {
        /* Probably not, as we couldn't attach to the parent process's
            console.
            Try to create a console.

            According to a comment on

http://msdn.microsoft.com/en-us/library/windows/desktop/ms681952(v=vs.85).aspx

            (which now redirects to a docs.microsoft.com page that is
            devoid of comments, and which is not available on the
            Wayback Machine)

            and according to

http://connect.microsoft.com/VisualStudio/feedback/details/689696/installing-security-update-kb2507938-prevents-console-allocation

            (which has disappeared, and isn't available on the Wayback
            Machine)

            and

https://answers.microsoft.com/en-us/windows/forum/windows_xp-windows_update/kb2567680-andor-kb2507938-breaks-attachconsole-api/e8191280-2d49-4be4-9918-18486fba0afa

even a failed attempt to attach to another process's console
will cause subsequent AllocConsole() calls to fail, possibly due
to bugs introduced by a security patch.  To work around this, we
do a FreeConsole() first. */
        FreeConsole();
        if (AllocConsole()) {
            /* That succeeded. */
            console_wait = true;
            SetConsoleTitle(_T("Wireshark Debug Console"));
        } else {
            /* On Windows XP, this still fails; FreeConsole() apparently
                doesn't clear the state, as it does on Windows 7. */
            return;   /* couldn't create console */
        }
    }

    if (must_redirect_stdin)
        ws_freopen("CONIN$", "r", stdin);
    if (must_redirect_stdout) {
        ws_freopen("CONOUT$", "w", stdout);
        fprintf(stdout, "\n");
    }
    if (must_redirect_stderr) {
        ws_freopen("CONOUT$", "w", stderr);
        fprintf(stderr, "\n");
    }

    /* Now register "destroy_console()" as a routine to be called just
        before the application exits, so that we can destroy the console
        after the user has typed a key (so that the console doesn't just
        disappear out from under them, giving the user no chance to see
        the message(s) we put in there). */
    atexit(destroy_console);

    /* Well, we have a console now. */
    has_console = true;
}

void
restore_pipes(void)
{
    bool must_redirect_stdin;
    bool must_redirect_stdout;
    bool must_redirect_stderr;

    HANDLE fd;

    if (stdin_capture) {
        /* We've been handed "-i -". Don't mess with stdio. */
        return;
    }

    if (has_console) {
        return;
    }

    /* Are the standard input, output, and error invalid handles? */
    must_redirect_stdin = needs_redirection(STD_INPUT_HANDLE);
    must_redirect_stdout = needs_redirection(STD_OUTPUT_HANDLE);
    must_redirect_stderr = needs_redirection(STD_ERROR_HANDLE);

    /* If none of them are invalid, we don't need to do anything. */
    if (!must_redirect_stdin && !must_redirect_stdout && !must_redirect_stderr)
        return;

    if (!must_redirect_stdin)
        fd = GetStdHandle(STD_INPUT_HANDLE);

    /* OK, at least one of them needs to be redirected to a console;
        try to attach to the parent process's console and, if that fails,
        cleanup and return. */
    /*
        * See if we have an existing console (i.e. we were run from a
        * command prompt).
        */
    if (!AttachConsole(ATTACH_PARENT_PROCESS)) {
        FreeConsole();
        return;   /* No parent - cleanup and exit */
    }

    if (must_redirect_stdin) {
        ws_freopen("CONIN$", "r", stdin);
    } else {
        SetStdHandle(STD_INPUT_HANDLE, fd);
    }
    if (must_redirect_stdout) {
        ws_freopen("CONOUT$", "w", stdout);
        fprintf(stdout, "\n");
    }
    if (must_redirect_stderr) {
        ws_freopen("CONOUT$", "w", stderr);
        fprintf(stderr, "\n");
    }

    /* Now register "destroy_console()" as a routine to be called just
        before the application exits, so that we can destroy the console
        after the user has typed a key (so that the console doesn't just
        disappear out from under them, giving the user no chance to see
        the message(s) we put in there). */
    atexit(destroy_console);

    /* Well, we have a console now. */
    has_console = true;
}

void
destroy_console(void)
{
    if (console_wait) {
        printf("\n\nPress any key to exit\n");
        _getch();
    }
    FreeConsole();
}

void
set_console_wait(bool set_console_wait)
{
    console_wait = set_console_wait;
}

bool
get_console_wait(void)
{
    return console_wait;
}

void
set_stdin_capture(bool set_stdin_capture)
{
    stdin_capture = set_stdin_capture;
}

bool
get_stdin_capture(void)
{
    return stdin_capture;
}

#endif /* _WIN32 */